diff --git a/.gitattributes b/.gitattributes
index a6344aac8c09253b3b630fb776ae94478aa0275b..c83d33ffd8e99064a4551c37f38a33959b77970d 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -33,3 +33,9 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+checkpoint-109/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoint-218/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoint-327/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoint-436/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoint-545/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text
diff --git a/adapter_config.json b/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..d7d7c196589a89c5b284ba682f27212262154f44
--- /dev/null
+++ b/adapter_config.json
@@ -0,0 +1,42 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "nvidia/Llama-3_3-Nemotron-Super-49B-v1",
+  "bias": "none",
+  "corda_config": null,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": null,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 512,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": [
+    "embed_tokens",
+    "lm_head"
+  ],
+  "peft_type": "LORA",
+  "r": 256,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "o_proj",
+    "gate_proj",
+    "q_proj",
+    "k_proj",
+    "v_proj",
+    "up_proj",
+    "down_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_rslora": false
+}
\ No newline at end of file
diff --git a/checkpoint-109/README.md b/checkpoint-109/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..f4a3934800eeb082a0cb833d7b6af4f68eed3615
--- /dev/null
+++ b/checkpoint-109/README.md
@@ -0,0 +1,202 @@
+---
+base_model: nvidia/Llama-3_3-Nemotron-Super-49B-v1
+library_name: peft
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.15.0
\ No newline at end of file
diff --git a/checkpoint-109/adapter_config.json b/checkpoint-109/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..d7d7c196589a89c5b284ba682f27212262154f44
--- /dev/null
+++ b/checkpoint-109/adapter_config.json
@@ -0,0 +1,42 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "nvidia/Llama-3_3-Nemotron-Super-49B-v1",
+  "bias": "none",
+  "corda_config": null,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": null,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 512,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": [
+    "embed_tokens",
+    "lm_head"
+  ],
+  "peft_type": "LORA",
+  "r": 256,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "o_proj",
+    "gate_proj",
+    "q_proj",
+    "k_proj",
+    "v_proj",
+    "up_proj",
+    "down_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_rslora": false
+}
\ No newline at end of file
diff --git a/checkpoint-109/adapter_model.safetensors b/checkpoint-109/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..fdba70f7e961ee577f3d0a98bbf775df6847b288
--- /dev/null
+++ b/checkpoint-109/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a7f32affafdc09c369bba68357ca31956e8086ce7701e04e04ea6756072bf79
+size 9016826528
diff --git a/checkpoint-109/global_step109/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/checkpoint-109/global_step109/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..76146a9c2adbc1344eae7ab106f28328fd79be47
--- /dev/null
+++ b/checkpoint-109/global_step109/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:279ec51cd173960a46dd5270a1aa392dc73ae653e4967424eea6f330d95891a6
+size 27050164444
diff --git a/checkpoint-109/global_step109/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/checkpoint-109/global_step109/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..847002d0f045e53a781ea34eda939aabb3ca943e
--- /dev/null
+++ b/checkpoint-109/global_step109/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ad9840509fc381eea5d1004bd82f0efc0f7640e851b0b457bb116718ad3bc4ba
+size 27050169884
diff --git a/checkpoint-109/global_step109/mp_rank_00_model_states.pt b/checkpoint-109/global_step109/mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9f58f048a401042fef7b6d766c2ff8a04e12ebd1
--- /dev/null
+++ b/checkpoint-109/global_step109/mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:163f84be3c633a9af23549b2f2ce50102524def58bf0d451800acd13c59b6cd6
+size 9776788601
diff --git a/checkpoint-109/latest b/checkpoint-109/latest
new file mode 100644
index 0000000000000000000000000000000000000000..98e330c5e53ebf58122f7ce0a9ce018fa94aea8a
--- /dev/null
+++ b/checkpoint-109/latest
@@ -0,0 +1 @@
+global_step109
\ No newline at end of file
diff --git a/checkpoint-109/rng_state_0.pth b/checkpoint-109/rng_state_0.pth
new file mode 100644
index 0000000000000000000000000000000000000000..9ee62bb84931c4a6e6ea7fd605de0051607bc562
--- /dev/null
+++ b/checkpoint-109/rng_state_0.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:43d83a11dba599c305208d485dc434bc95abca3c8b9c33dc1de687c354f3ef0d
+size 14512
diff --git a/checkpoint-109/rng_state_1.pth b/checkpoint-109/rng_state_1.pth
new file mode 100644
index 0000000000000000000000000000000000000000..1029bc80f46f56f1f70b7d756e3d843b52d85965
--- /dev/null
+++ b/checkpoint-109/rng_state_1.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f5b35da1af2f35f904ecb92a644c838c9e9006e646cc823d7d615ed5cc45d4e0
+size 14512
diff --git a/checkpoint-109/scheduler.pt b/checkpoint-109/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..418d9708dd56b59b4fe9b88e62f1537e9ae8c12a
--- /dev/null
+++ b/checkpoint-109/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d452b3b665e980d65823f2d1807a5ce26dba0d3a7ea072f6413bae99f16bc430
+size 1064
diff --git a/checkpoint-109/special_tokens_map.json b/checkpoint-109/special_tokens_map.json
new file mode 100644
index 0000000000000000000000000000000000000000..278b7f0f84be865c4687700ee7b3c63d89a51e18
--- /dev/null
+++ b/checkpoint-109/special_tokens_map.json
@@ -0,0 +1,23 @@
+{
+  "bos_token": {
+    "content": "<|begin_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|eot_id|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|end_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}
diff --git a/checkpoint-109/tokenizer.json b/checkpoint-109/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..1c1d8d5c9024994f1d3b00f9662b8dd89ca13cf2
--- /dev/null
+++ b/checkpoint-109/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920
diff --git a/checkpoint-109/tokenizer_config.json b/checkpoint-109/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..edd01b980c1db496ea102a51c972ee8f5d1a2c74
--- /dev/null
+++ b/checkpoint-109/tokenizer_config.json
@@ -0,0 +1,2064 @@
+{
+  "added_tokens_decoder": {
+    "128000": {
+      "content": "<|begin_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128001": {
+      "content": "<|end_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128002": {
+      "content": "<|reserved_special_token_0|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128003": {
+      "content": "<|reserved_special_token_1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128004": {
+      "content": "<|finetune_right_pad_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128005": {
+      "content": "<|reserved_special_token_2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128006": {
+      "content": "<|start_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128007": {
+      "content": "<|end_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128008": {
+      "content": "<|eom_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128009": {
+      "content": "<|eot_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128010": {
+      "content": "<|python_tag|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128011": {
+      "content": "<|reserved_special_token_3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128012": {
+      "content": "<|reserved_special_token_4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128013": {
+      "content": "<|reserved_special_token_5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128014": {
+      "content": "<|reserved_special_token_6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128015": {
+      "content": "<|reserved_special_token_7|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128016": {
+      "content": "<|reserved_special_token_8|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128017": {
+      "content": "<|reserved_special_token_9|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128018": {
+      "content": "<|reserved_special_token_10|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128019": {
+      "content": "<|reserved_special_token_11|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128020": {
+      "content": "<|reserved_special_token_12|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128021": {
+      "content": "<|reserved_special_token_13|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128022": {
+      "content": "<|reserved_special_token_14|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128023": {
+      "content": "<|reserved_special_token_15|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128024": {
+      "content": "<|reserved_special_token_16|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128025": {
+      "content": "<|reserved_special_token_17|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128026": {
+      "content": "<|reserved_special_token_18|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128027": {
+      "content": "<|reserved_special_token_19|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128028": {
+      "content": "<|reserved_special_token_20|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128029": {
+      "content": "<|reserved_special_token_21|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128030": {
+      "content": "<|reserved_special_token_22|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128031": {
+      "content": "<|reserved_special_token_23|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128032": {
+      "content": "<|reserved_special_token_24|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128033": {
+      "content": "<|reserved_special_token_25|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128034": {
+      "content": "<|reserved_special_token_26|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128035": {
+      "content": "<|reserved_special_token_27|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128036": {
+      "content": "<|reserved_special_token_28|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128037": {
+      "content": "<|reserved_special_token_29|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128038": {
+      "content": "<|reserved_special_token_30|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128039": {
+      "content": "<|reserved_special_token_31|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128040": {
+      "content": "<|reserved_special_token_32|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128041": {
+      "content": "<|reserved_special_token_33|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128042": {
+      "content": "<|reserved_special_token_34|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128043": {
+      "content": "<|reserved_special_token_35|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128044": {
+      "content": "<|reserved_special_token_36|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128045": {
+      "content": "<|reserved_special_token_37|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128046": {
+      "content": "<|reserved_special_token_38|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128047": {
+      "content": "<|reserved_special_token_39|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128048": {
+      "content": "<|reserved_special_token_40|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128049": {
+      "content": "<|reserved_special_token_41|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128050": {
+      "content": "<|reserved_special_token_42|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128051": {
+      "content": "<|reserved_special_token_43|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128052": {
+      "content": "<|reserved_special_token_44|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128053": {
+      "content": "<|reserved_special_token_45|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128054": {
+      "content": "<|reserved_special_token_46|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128055": {
+      "content": "<|reserved_special_token_47|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128056": {
+      "content": "<|reserved_special_token_48|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128057": {
+      "content": "<|reserved_special_token_49|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128058": {
+      "content": "<|reserved_special_token_50|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128059": {
+      "content": "<|reserved_special_token_51|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128060": {
+      "content": "<|reserved_special_token_52|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128061": {
+      "content": "<|reserved_special_token_53|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128062": {
+      "content": "<|reserved_special_token_54|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128063": {
+      "content": "<|reserved_special_token_55|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128064": {
+      "content": "<|reserved_special_token_56|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128065": {
+      "content": "<|reserved_special_token_57|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128066": {
+      "content": "<|reserved_special_token_58|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128067": {
+      "content": "<|reserved_special_token_59|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128068": {
+      "content": "<|reserved_special_token_60|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128069": {
+      "content": "<|reserved_special_token_61|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128070": {
+      "content": "<|reserved_special_token_62|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128071": {
+      "content": "<|reserved_special_token_63|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128072": {
+      "content": "<|reserved_special_token_64|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128073": {
+      "content": "<|reserved_special_token_65|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128074": {
+      "content": "<|reserved_special_token_66|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128075": {
+      "content": "<|reserved_special_token_67|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128076": {
+      "content": "<|reserved_special_token_68|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128077": {
+      "content": "<|reserved_special_token_69|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128078": {
+      "content": "<|reserved_special_token_70|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128079": {
+      "content": "<|reserved_special_token_71|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128080": {
+      "content": "<|reserved_special_token_72|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128081": {
+      "content": "<|reserved_special_token_73|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128082": {
+      "content": "<|reserved_special_token_74|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128083": {
+      "content": "<|reserved_special_token_75|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128084": {
+      "content": "<|reserved_special_token_76|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128085": {
+      "content": "<|reserved_special_token_77|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128086": {
+      "content": "<|reserved_special_token_78|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128087": {
+      "content": "<|reserved_special_token_79|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128088": {
+      "content": "<|reserved_special_token_80|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128089": {
+      "content": "<|reserved_special_token_81|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128090": {
+      "content": "<|reserved_special_token_82|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128091": {
+      "content": "<|reserved_special_token_83|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128092": {
+      "content": "<|reserved_special_token_84|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128093": {
+      "content": "<|reserved_special_token_85|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128094": {
+      "content": "<|reserved_special_token_86|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128095": {
+      "content": "<|reserved_special_token_87|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128096": {
+      "content": "<|reserved_special_token_88|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128097": {
+      "content": "<|reserved_special_token_89|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128098": {
+      "content": "<|reserved_special_token_90|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128099": {
+      "content": "<|reserved_special_token_91|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128100": {
+      "content": "<|reserved_special_token_92|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128101": {
+      "content": "<|reserved_special_token_93|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128102": {
+      "content": "<|reserved_special_token_94|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128103": {
+      "content": "<|reserved_special_token_95|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128104": {
+      "content": "<|reserved_special_token_96|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128105": {
+      "content": "<|reserved_special_token_97|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128106": {
+      "content": "<|reserved_special_token_98|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128107": {
+      "content": "<|reserved_special_token_99|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128108": {
+      "content": "<|reserved_special_token_100|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128109": {
+      "content": "<|reserved_special_token_101|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128110": {
+      "content": "<|reserved_special_token_102|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128111": {
+      "content": "<|reserved_special_token_103|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128112": {
+      "content": "<|reserved_special_token_104|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128113": {
+      "content": "<|reserved_special_token_105|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128114": {
+      "content": "<|reserved_special_token_106|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128115": {
+      "content": "<|reserved_special_token_107|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128116": {
+      "content": "<|reserved_special_token_108|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128117": {
+      "content": "<|reserved_special_token_109|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128118": {
+      "content": "<|reserved_special_token_110|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128119": {
+      "content": "<|reserved_special_token_111|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128120": {
+      "content": "<|reserved_special_token_112|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128121": {
+      "content": "<|reserved_special_token_113|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128122": {
+      "content": "<|reserved_special_token_114|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128123": {
+      "content": "<|reserved_special_token_115|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128124": {
+      "content": "<|reserved_special_token_116|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128125": {
+      "content": "<|reserved_special_token_117|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128126": {
+      "content": "<|reserved_special_token_118|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128127": {
+      "content": "<|reserved_special_token_119|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128128": {
+      "content": "<|reserved_special_token_120|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128129": {
+      "content": "<|reserved_special_token_121|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128130": {
+      "content": "<|reserved_special_token_122|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128131": {
+      "content": "<|reserved_special_token_123|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128132": {
+      "content": "<|reserved_special_token_124|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128133": {
+      "content": "<|reserved_special_token_125|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128134": {
+      "content": "<|reserved_special_token_126|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128135": {
+      "content": "<|reserved_special_token_127|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128136": {
+      "content": "<|reserved_special_token_128|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128137": {
+      "content": "<|reserved_special_token_129|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128138": {
+      "content": "<|reserved_special_token_130|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128139": {
+      "content": "<|reserved_special_token_131|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128140": {
+      "content": "<|reserved_special_token_132|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128141": {
+      "content": "<|reserved_special_token_133|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128142": {
+      "content": "<|reserved_special_token_134|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128143": {
+      "content": "<|reserved_special_token_135|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128144": {
+      "content": "<|reserved_special_token_136|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128145": {
+      "content": "<|reserved_special_token_137|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128146": {
+      "content": "<|reserved_special_token_138|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128147": {
+      "content": "<|reserved_special_token_139|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128148": {
+      "content": "<|reserved_special_token_140|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128149": {
+      "content": "<|reserved_special_token_141|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128150": {
+      "content": "<|reserved_special_token_142|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128151": {
+      "content": "<|reserved_special_token_143|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128152": {
+      "content": "<|reserved_special_token_144|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128153": {
+      "content": "<|reserved_special_token_145|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128154": {
+      "content": "<|reserved_special_token_146|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128155": {
+      "content": "<|reserved_special_token_147|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128156": {
+      "content": "<|reserved_special_token_148|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128157": {
+      "content": "<|reserved_special_token_149|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128158": {
+      "content": "<|reserved_special_token_150|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128159": {
+      "content": "<|reserved_special_token_151|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128160": {
+      "content": "<|reserved_special_token_152|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128161": {
+      "content": "<|reserved_special_token_153|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128162": {
+      "content": "<|reserved_special_token_154|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128163": {
+      "content": "<|reserved_special_token_155|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128164": {
+      "content": "<|reserved_special_token_156|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128165": {
+      "content": "<|reserved_special_token_157|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128166": {
+      "content": "<|reserved_special_token_158|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128167": {
+      "content": "<|reserved_special_token_159|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128168": {
+      "content": "<|reserved_special_token_160|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128169": {
+      "content": "<|reserved_special_token_161|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128170": {
+      "content": "<|reserved_special_token_162|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128171": {
+      "content": "<|reserved_special_token_163|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128172": {
+      "content": "<|reserved_special_token_164|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128173": {
+      "content": "<|reserved_special_token_165|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128174": {
+      "content": "<|reserved_special_token_166|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128175": {
+      "content": "<|reserved_special_token_167|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128176": {
+      "content": "<|reserved_special_token_168|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128177": {
+      "content": "<|reserved_special_token_169|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128178": {
+      "content": "<|reserved_special_token_170|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128179": {
+      "content": "<|reserved_special_token_171|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128180": {
+      "content": "<|reserved_special_token_172|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128181": {
+      "content": "<|reserved_special_token_173|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128182": {
+      "content": "<|reserved_special_token_174|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128183": {
+      "content": "<|reserved_special_token_175|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128184": {
+      "content": "<|reserved_special_token_176|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128185": {
+      "content": "<|reserved_special_token_177|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128186": {
+      "content": "<|reserved_special_token_178|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128187": {
+      "content": "<|reserved_special_token_179|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128188": {
+      "content": "<|reserved_special_token_180|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128189": {
+      "content": "<|reserved_special_token_181|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128190": {
+      "content": "<|reserved_special_token_182|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128191": {
+      "content": "<|reserved_special_token_183|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128192": {
+      "content": "<|reserved_special_token_184|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128193": {
+      "content": "<|reserved_special_token_185|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128194": {
+      "content": "<|reserved_special_token_186|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128195": {
+      "content": "<|reserved_special_token_187|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128196": {
+      "content": "<|reserved_special_token_188|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128197": {
+      "content": "<|reserved_special_token_189|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128198": {
+      "content": "<|reserved_special_token_190|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128199": {
+      "content": "<|reserved_special_token_191|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128200": {
+      "content": "<|reserved_special_token_192|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128201": {
+      "content": "<|reserved_special_token_193|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128202": {
+      "content": "<|reserved_special_token_194|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128203": {
+      "content": "<|reserved_special_token_195|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128204": {
+      "content": "<|reserved_special_token_196|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128205": {
+      "content": "<|reserved_special_token_197|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128206": {
+      "content": "<|reserved_special_token_198|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128207": {
+      "content": "<|reserved_special_token_199|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128208": {
+      "content": "<|reserved_special_token_200|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128209": {
+      "content": "<|reserved_special_token_201|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128210": {
+      "content": "<|reserved_special_token_202|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128211": {
+      "content": "<|reserved_special_token_203|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128212": {
+      "content": "<|reserved_special_token_204|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128213": {
+      "content": "<|reserved_special_token_205|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128214": {
+      "content": "<|reserved_special_token_206|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128215": {
+      "content": "<|reserved_special_token_207|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128216": {
+      "content": "<|reserved_special_token_208|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128217": {
+      "content": "<|reserved_special_token_209|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128218": {
+      "content": "<|reserved_special_token_210|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128219": {
+      "content": "<|reserved_special_token_211|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128220": {
+      "content": "<|reserved_special_token_212|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128221": {
+      "content": "<|reserved_special_token_213|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128222": {
+      "content": "<|reserved_special_token_214|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128223": {
+      "content": "<|reserved_special_token_215|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128224": {
+      "content": "<|reserved_special_token_216|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128225": {
+      "content": "<|reserved_special_token_217|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128226": {
+      "content": "<|reserved_special_token_218|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128227": {
+      "content": "<|reserved_special_token_219|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128228": {
+      "content": "<|reserved_special_token_220|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128229": {
+      "content": "<|reserved_special_token_221|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128230": {
+      "content": "<|reserved_special_token_222|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128231": {
+      "content": "<|reserved_special_token_223|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128232": {
+      "content": "<|reserved_special_token_224|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128233": {
+      "content": "<|reserved_special_token_225|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128234": {
+      "content": "<|reserved_special_token_226|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128235": {
+      "content": "<|reserved_special_token_227|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128236": {
+      "content": "<|reserved_special_token_228|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128237": {
+      "content": "<|reserved_special_token_229|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128238": {
+      "content": "<|reserved_special_token_230|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128239": {
+      "content": "<|reserved_special_token_231|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128240": {
+      "content": "<|reserved_special_token_232|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128241": {
+      "content": "<|reserved_special_token_233|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128242": {
+      "content": "<|reserved_special_token_234|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128243": {
+      "content": "<|reserved_special_token_235|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128244": {
+      "content": "<|reserved_special_token_236|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128245": {
+      "content": "<|reserved_special_token_237|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128246": {
+      "content": "<|reserved_special_token_238|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128247": {
+      "content": "<|reserved_special_token_239|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128248": {
+      "content": "<|reserved_special_token_240|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128249": {
+      "content": "<|reserved_special_token_241|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128250": {
+      "content": "<|reserved_special_token_242|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128251": {
+      "content": "<|reserved_special_token_243|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128252": {
+      "content": "<|reserved_special_token_244|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128253": {
+      "content": "<|reserved_special_token_245|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128254": {
+      "content": "<|reserved_special_token_246|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128255": {
+      "content": "<|reserved_special_token_247|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|begin_of_text|>",
+  "chat_template": "{{- bos_token }}{%- if messages[0]['role'] == 'system' %}{%- set system_message = messages[0]['content']|trim %}{%- set messages = messages[1:] %}{%- else %}{%- set system_message = \"\" %}{%- endif %}{{- \"<|start_header_id|>system<|end_header_id|>\\n\\n\" }}{{- system_message }}{{- \"<|eot_id|>\" }}{%- for message in messages %}{%- if message['role'] == 'assistant' and '</think>' in message['content'] %}{%- set content = message['content'].split('</think>')[-1].lstrip() %}{%- else %}{%- set content = message['content'] %}{%- endif %}{{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\\n\\n' + content | trim + '<|eot_id|>' }}{%- endfor %}{%- if add_generation_prompt %}{{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' }}{%- endif %}",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|eot_id|>",
+  "extra_special_tokens": {},
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "pad_token": "<|end_of_text|>",
+  "tokenizer_class": "PreTrainedTokenizer"
+}
diff --git a/checkpoint-109/trainer_state.json b/checkpoint-109/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..539a86b3d37977d780b426c72dce26888b9afb41
--- /dev/null
+++ b/checkpoint-109/trainer_state.json
@@ -0,0 +1,796 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.9969512195121951,
+  "eval_steps": 500,
+  "global_step": 109,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.009146341463414634,
+      "grad_norm": 40.05127716064453,
+      "learning_rate": 5.0000000000000004e-08,
+      "loss": 5.1282,
+      "step": 1
+    },
+    {
+      "epoch": 0.018292682926829267,
+      "grad_norm": 39.01039505004883,
+      "learning_rate": 1.0000000000000001e-07,
+      "loss": 5.0586,
+      "step": 2
+    },
+    {
+      "epoch": 0.027439024390243903,
+      "grad_norm": 40.20766067504883,
+      "learning_rate": 1.5000000000000002e-07,
+      "loss": 5.0311,
+      "step": 3
+    },
+    {
+      "epoch": 0.036585365853658534,
+      "grad_norm": 38.70350646972656,
+      "learning_rate": 2.0000000000000002e-07,
+      "loss": 5.1598,
+      "step": 4
+    },
+    {
+      "epoch": 0.04573170731707317,
+      "grad_norm": 39.230979919433594,
+      "learning_rate": 2.5000000000000004e-07,
+      "loss": 5.0812,
+      "step": 5
+    },
+    {
+      "epoch": 0.054878048780487805,
+      "grad_norm": 39.45293045043945,
+      "learning_rate": 3.0000000000000004e-07,
+      "loss": 5.0841,
+      "step": 6
+    },
+    {
+      "epoch": 0.06402439024390244,
+      "grad_norm": 38.92170333862305,
+      "learning_rate": 3.5000000000000004e-07,
+      "loss": 5.022,
+      "step": 7
+    },
+    {
+      "epoch": 0.07317073170731707,
+      "grad_norm": 38.932369232177734,
+      "learning_rate": 4.0000000000000003e-07,
+      "loss": 5.0876,
+      "step": 8
+    },
+    {
+      "epoch": 0.08231707317073171,
+      "grad_norm": 39.60245132446289,
+      "learning_rate": 4.5000000000000003e-07,
+      "loss": 5.0798,
+      "step": 9
+    },
+    {
+      "epoch": 0.09146341463414634,
+      "grad_norm": 39.53607177734375,
+      "learning_rate": 5.000000000000001e-07,
+      "loss": 5.0105,
+      "step": 10
+    },
+    {
+      "epoch": 0.10060975609756098,
+      "grad_norm": 39.50700378417969,
+      "learning_rate": 5.5e-07,
+      "loss": 4.9951,
+      "step": 11
+    },
+    {
+      "epoch": 0.10975609756097561,
+      "grad_norm": 39.52376174926758,
+      "learning_rate": 6.000000000000001e-07,
+      "loss": 4.9163,
+      "step": 12
+    },
+    {
+      "epoch": 0.11890243902439024,
+      "grad_norm": 39.15214157104492,
+      "learning_rate": 6.5e-07,
+      "loss": 4.9634,
+      "step": 13
+    },
+    {
+      "epoch": 0.12804878048780488,
+      "grad_norm": 37.795066833496094,
+      "learning_rate": 7.000000000000001e-07,
+      "loss": 4.819,
+      "step": 14
+    },
+    {
+      "epoch": 0.13719512195121952,
+      "grad_norm": 38.23014450073242,
+      "learning_rate": 7.5e-07,
+      "loss": 4.8756,
+      "step": 15
+    },
+    {
+      "epoch": 0.14634146341463414,
+      "grad_norm": 38.57551574707031,
+      "learning_rate": 8.000000000000001e-07,
+      "loss": 4.81,
+      "step": 16
+    },
+    {
+      "epoch": 0.15548780487804878,
+      "grad_norm": 36.06121063232422,
+      "learning_rate": 8.500000000000001e-07,
+      "loss": 4.7901,
+      "step": 17
+    },
+    {
+      "epoch": 0.16463414634146342,
+      "grad_norm": 34.38008117675781,
+      "learning_rate": 9.000000000000001e-07,
+      "loss": 4.5775,
+      "step": 18
+    },
+    {
+      "epoch": 0.17378048780487804,
+      "grad_norm": 32.60890579223633,
+      "learning_rate": 9.500000000000001e-07,
+      "loss": 4.3737,
+      "step": 19
+    },
+    {
+      "epoch": 0.18292682926829268,
+      "grad_norm": 31.983110427856445,
+      "learning_rate": 1.0000000000000002e-06,
+      "loss": 4.3773,
+      "step": 20
+    },
+    {
+      "epoch": 0.19207317073170732,
+      "grad_norm": 27.69571876525879,
+      "learning_rate": 1.0500000000000001e-06,
+      "loss": 4.1276,
+      "step": 21
+    },
+    {
+      "epoch": 0.20121951219512196,
+      "grad_norm": 26.542314529418945,
+      "learning_rate": 1.1e-06,
+      "loss": 4.0641,
+      "step": 22
+    },
+    {
+      "epoch": 0.21036585365853658,
+      "grad_norm": 23.26603126525879,
+      "learning_rate": 1.1500000000000002e-06,
+      "loss": 4.0197,
+      "step": 23
+    },
+    {
+      "epoch": 0.21951219512195122,
+      "grad_norm": 21.04385757446289,
+      "learning_rate": 1.2000000000000002e-06,
+      "loss": 4.0179,
+      "step": 24
+    },
+    {
+      "epoch": 0.22865853658536586,
+      "grad_norm": 19.790103912353516,
+      "learning_rate": 1.25e-06,
+      "loss": 3.8334,
+      "step": 25
+    },
+    {
+      "epoch": 0.23780487804878048,
+      "grad_norm": 18.2396183013916,
+      "learning_rate": 1.3e-06,
+      "loss": 3.7027,
+      "step": 26
+    },
+    {
+      "epoch": 0.24695121951219512,
+      "grad_norm": 18.725860595703125,
+      "learning_rate": 1.3500000000000002e-06,
+      "loss": 3.7268,
+      "step": 27
+    },
+    {
+      "epoch": 0.25609756097560976,
+      "grad_norm": 18.91561508178711,
+      "learning_rate": 1.4000000000000001e-06,
+      "loss": 3.52,
+      "step": 28
+    },
+    {
+      "epoch": 0.2652439024390244,
+      "grad_norm": 18.196651458740234,
+      "learning_rate": 1.45e-06,
+      "loss": 3.4765,
+      "step": 29
+    },
+    {
+      "epoch": 0.27439024390243905,
+      "grad_norm": 17.778606414794922,
+      "learning_rate": 1.5e-06,
+      "loss": 3.3573,
+      "step": 30
+    },
+    {
+      "epoch": 0.28353658536585363,
+      "grad_norm": 16.23012924194336,
+      "learning_rate": 1.5500000000000002e-06,
+      "loss": 3.2783,
+      "step": 31
+    },
+    {
+      "epoch": 0.2926829268292683,
+      "grad_norm": 15.836565017700195,
+      "learning_rate": 1.6000000000000001e-06,
+      "loss": 3.2319,
+      "step": 32
+    },
+    {
+      "epoch": 0.3018292682926829,
+      "grad_norm": 11.919289588928223,
+      "learning_rate": 1.6500000000000003e-06,
+      "loss": 3.1345,
+      "step": 33
+    },
+    {
+      "epoch": 0.31097560975609756,
+      "grad_norm": 11.086843490600586,
+      "learning_rate": 1.7000000000000002e-06,
+      "loss": 3.1153,
+      "step": 34
+    },
+    {
+      "epoch": 0.3201219512195122,
+      "grad_norm": 7.563486576080322,
+      "learning_rate": 1.75e-06,
+      "loss": 2.9784,
+      "step": 35
+    },
+    {
+      "epoch": 0.32926829268292684,
+      "grad_norm": 7.60545539855957,
+      "learning_rate": 1.8000000000000001e-06,
+      "loss": 2.8568,
+      "step": 36
+    },
+    {
+      "epoch": 0.3384146341463415,
+      "grad_norm": 7.071218013763428,
+      "learning_rate": 1.85e-06,
+      "loss": 2.8879,
+      "step": 37
+    },
+    {
+      "epoch": 0.3475609756097561,
+      "grad_norm": 8.200672149658203,
+      "learning_rate": 1.9000000000000002e-06,
+      "loss": 2.7437,
+      "step": 38
+    },
+    {
+      "epoch": 0.3567073170731707,
+      "grad_norm": 12.079374313354492,
+      "learning_rate": 1.9500000000000004e-06,
+      "loss": 2.7589,
+      "step": 39
+    },
+    {
+      "epoch": 0.36585365853658536,
+      "grad_norm": 12.21938705444336,
+      "learning_rate": 2.0000000000000003e-06,
+      "loss": 2.7098,
+      "step": 40
+    },
+    {
+      "epoch": 0.375,
+      "grad_norm": 10.224228858947754,
+      "learning_rate": 2.05e-06,
+      "loss": 2.6803,
+      "step": 41
+    },
+    {
+      "epoch": 0.38414634146341464,
+      "grad_norm": 10.054616928100586,
+      "learning_rate": 2.1000000000000002e-06,
+      "loss": 2.4658,
+      "step": 42
+    },
+    {
+      "epoch": 0.3932926829268293,
+      "grad_norm": 8.074049949645996,
+      "learning_rate": 2.15e-06,
+      "loss": 2.6341,
+      "step": 43
+    },
+    {
+      "epoch": 0.4024390243902439,
+      "grad_norm": 5.721987724304199,
+      "learning_rate": 2.2e-06,
+      "loss": 2.3848,
+      "step": 44
+    },
+    {
+      "epoch": 0.4115853658536585,
+      "grad_norm": 4.646566390991211,
+      "learning_rate": 2.25e-06,
+      "loss": 2.4323,
+      "step": 45
+    },
+    {
+      "epoch": 0.42073170731707316,
+      "grad_norm": 4.591104984283447,
+      "learning_rate": 2.3000000000000004e-06,
+      "loss": 2.4022,
+      "step": 46
+    },
+    {
+      "epoch": 0.4298780487804878,
+      "grad_norm": 4.822715759277344,
+      "learning_rate": 2.35e-06,
+      "loss": 2.2768,
+      "step": 47
+    },
+    {
+      "epoch": 0.43902439024390244,
+      "grad_norm": 5.176568984985352,
+      "learning_rate": 2.4000000000000003e-06,
+      "loss": 2.281,
+      "step": 48
+    },
+    {
+      "epoch": 0.4481707317073171,
+      "grad_norm": 4.731242656707764,
+      "learning_rate": 2.4500000000000003e-06,
+      "loss": 2.235,
+      "step": 49
+    },
+    {
+      "epoch": 0.4573170731707317,
+      "grad_norm": 4.6343865394592285,
+      "learning_rate": 2.5e-06,
+      "loss": 2.1465,
+      "step": 50
+    },
+    {
+      "epoch": 0.46646341463414637,
+      "grad_norm": 4.669382095336914,
+      "learning_rate": 2.55e-06,
+      "loss": 2.0969,
+      "step": 51
+    },
+    {
+      "epoch": 0.47560975609756095,
+      "grad_norm": 4.207769870758057,
+      "learning_rate": 2.6e-06,
+      "loss": 2.0308,
+      "step": 52
+    },
+    {
+      "epoch": 0.4847560975609756,
+      "grad_norm": 5.222470760345459,
+      "learning_rate": 2.6500000000000005e-06,
+      "loss": 1.9445,
+      "step": 53
+    },
+    {
+      "epoch": 0.49390243902439024,
+      "grad_norm": 5.10925817489624,
+      "learning_rate": 2.7000000000000004e-06,
+      "loss": 1.7763,
+      "step": 54
+    },
+    {
+      "epoch": 0.5030487804878049,
+      "grad_norm": 4.8151044845581055,
+      "learning_rate": 2.7500000000000004e-06,
+      "loss": 1.7661,
+      "step": 55
+    },
+    {
+      "epoch": 0.5121951219512195,
+      "grad_norm": 6.155700206756592,
+      "learning_rate": 2.8000000000000003e-06,
+      "loss": 1.6665,
+      "step": 56
+    },
+    {
+      "epoch": 0.5213414634146342,
+      "grad_norm": 4.719021320343018,
+      "learning_rate": 2.85e-06,
+      "loss": 1.5532,
+      "step": 57
+    },
+    {
+      "epoch": 0.5304878048780488,
+      "grad_norm": 4.591799259185791,
+      "learning_rate": 2.9e-06,
+      "loss": 1.5244,
+      "step": 58
+    },
+    {
+      "epoch": 0.5396341463414634,
+      "grad_norm": 5.483795642852783,
+      "learning_rate": 2.95e-06,
+      "loss": 1.3816,
+      "step": 59
+    },
+    {
+      "epoch": 0.5487804878048781,
+      "grad_norm": 4.32553768157959,
+      "learning_rate": 3e-06,
+      "loss": 1.4268,
+      "step": 60
+    },
+    {
+      "epoch": 0.5579268292682927,
+      "grad_norm": 8.296231269836426,
+      "learning_rate": 3.05e-06,
+      "loss": 1.2887,
+      "step": 61
+    },
+    {
+      "epoch": 0.5670731707317073,
+      "grad_norm": 5.07594108581543,
+      "learning_rate": 3.1000000000000004e-06,
+      "loss": 1.228,
+      "step": 62
+    },
+    {
+      "epoch": 0.5762195121951219,
+      "grad_norm": 8.427435874938965,
+      "learning_rate": 3.1500000000000003e-06,
+      "loss": 1.1724,
+      "step": 63
+    },
+    {
+      "epoch": 0.5853658536585366,
+      "grad_norm": 22.45092010498047,
+      "learning_rate": 3.2000000000000003e-06,
+      "loss": 1.0871,
+      "step": 64
+    },
+    {
+      "epoch": 0.5945121951219512,
+      "grad_norm": 20.67412567138672,
+      "learning_rate": 3.2500000000000002e-06,
+      "loss": 1.0108,
+      "step": 65
+    },
+    {
+      "epoch": 0.6036585365853658,
+      "grad_norm": 4.921634674072266,
+      "learning_rate": 3.3000000000000006e-06,
+      "loss": 0.9775,
+      "step": 66
+    },
+    {
+      "epoch": 0.6128048780487805,
+      "grad_norm": 4.477286338806152,
+      "learning_rate": 3.3500000000000005e-06,
+      "loss": 0.9554,
+      "step": 67
+    },
+    {
+      "epoch": 0.6219512195121951,
+      "grad_norm": 8.190194129943848,
+      "learning_rate": 3.4000000000000005e-06,
+      "loss": 0.8583,
+      "step": 68
+    },
+    {
+      "epoch": 0.6310975609756098,
+      "grad_norm": 29.21201515197754,
+      "learning_rate": 3.45e-06,
+      "loss": 0.8499,
+      "step": 69
+    },
+    {
+      "epoch": 0.6402439024390244,
+      "grad_norm": 33.81047439575195,
+      "learning_rate": 3.5e-06,
+      "loss": 0.858,
+      "step": 70
+    },
+    {
+      "epoch": 0.649390243902439,
+      "grad_norm": 3.5125606060028076,
+      "learning_rate": 3.5500000000000003e-06,
+      "loss": 0.7227,
+      "step": 71
+    },
+    {
+      "epoch": 0.6585365853658537,
+      "grad_norm": 18.493423461914062,
+      "learning_rate": 3.6000000000000003e-06,
+      "loss": 0.7447,
+      "step": 72
+    },
+    {
+      "epoch": 0.6676829268292683,
+      "grad_norm": 24.923927307128906,
+      "learning_rate": 3.65e-06,
+      "loss": 0.8423,
+      "step": 73
+    },
+    {
+      "epoch": 0.676829268292683,
+      "grad_norm": 9.870631217956543,
+      "learning_rate": 3.7e-06,
+      "loss": 0.6934,
+      "step": 74
+    },
+    {
+      "epoch": 0.6859756097560976,
+      "grad_norm": 20.598569869995117,
+      "learning_rate": 3.7500000000000005e-06,
+      "loss": 0.7005,
+      "step": 75
+    },
+    {
+      "epoch": 0.6951219512195121,
+      "grad_norm": 30.662466049194336,
+      "learning_rate": 3.8000000000000005e-06,
+      "loss": 0.6226,
+      "step": 76
+    },
+    {
+      "epoch": 0.7042682926829268,
+      "grad_norm": 28.930774688720703,
+      "learning_rate": 3.85e-06,
+      "loss": 0.6357,
+      "step": 77
+    },
+    {
+      "epoch": 0.7134146341463414,
+      "grad_norm": 7.140467166900635,
+      "learning_rate": 3.900000000000001e-06,
+      "loss": 0.6129,
+      "step": 78
+    },
+    {
+      "epoch": 0.7225609756097561,
+      "grad_norm": 6.322911739349365,
+      "learning_rate": 3.95e-06,
+      "loss": 0.6086,
+      "step": 79
+    },
+    {
+      "epoch": 0.7317073170731707,
+      "grad_norm": 5.404141426086426,
+      "learning_rate": 4.000000000000001e-06,
+      "loss": 0.5986,
+      "step": 80
+    },
+    {
+      "epoch": 0.7408536585365854,
+      "grad_norm": 24.445228576660156,
+      "learning_rate": 4.05e-06,
+      "loss": 0.5932,
+      "step": 81
+    },
+    {
+      "epoch": 0.75,
+      "grad_norm": 14.275381088256836,
+      "learning_rate": 4.1e-06,
+      "loss": 0.5286,
+      "step": 82
+    },
+    {
+      "epoch": 0.7591463414634146,
+      "grad_norm": 9.77132511138916,
+      "learning_rate": 4.15e-06,
+      "loss": 0.4814,
+      "step": 83
+    },
+    {
+      "epoch": 0.7682926829268293,
+      "grad_norm": 11.043930053710938,
+      "learning_rate": 4.2000000000000004e-06,
+      "loss": 0.4953,
+      "step": 84
+    },
+    {
+      "epoch": 0.7774390243902439,
+      "grad_norm": 3.1333775520324707,
+      "learning_rate": 4.25e-06,
+      "loss": 0.4794,
+      "step": 85
+    },
+    {
+      "epoch": 0.7865853658536586,
+      "grad_norm": 9.50448989868164,
+      "learning_rate": 4.3e-06,
+      "loss": 0.4451,
+      "step": 86
+    },
+    {
+      "epoch": 0.7957317073170732,
+      "grad_norm": 5.867074489593506,
+      "learning_rate": 4.350000000000001e-06,
+      "loss": 0.4224,
+      "step": 87
+    },
+    {
+      "epoch": 0.8048780487804879,
+      "grad_norm": 12.556106567382812,
+      "learning_rate": 4.4e-06,
+      "loss": 0.4224,
+      "step": 88
+    },
+    {
+      "epoch": 0.8140243902439024,
+      "grad_norm": 7.916622638702393,
+      "learning_rate": 4.450000000000001e-06,
+      "loss": 0.4091,
+      "step": 89
+    },
+    {
+      "epoch": 0.823170731707317,
+      "grad_norm": 2.970932722091675,
+      "learning_rate": 4.5e-06,
+      "loss": 0.4181,
+      "step": 90
+    },
+    {
+      "epoch": 0.8323170731707317,
+      "grad_norm": 17.468870162963867,
+      "learning_rate": 4.5500000000000005e-06,
+      "loss": 0.4031,
+      "step": 91
+    },
+    {
+      "epoch": 0.8414634146341463,
+      "grad_norm": 15.861083984375,
+      "learning_rate": 4.600000000000001e-06,
+      "loss": 0.3604,
+      "step": 92
+    },
+    {
+      "epoch": 0.850609756097561,
+      "grad_norm": 6.715958118438721,
+      "learning_rate": 4.65e-06,
+      "loss": 0.3489,
+      "step": 93
+    },
+    {
+      "epoch": 0.8597560975609756,
+      "grad_norm": 5.439844608306885,
+      "learning_rate": 4.7e-06,
+      "loss": 0.3339,
+      "step": 94
+    },
+    {
+      "epoch": 0.8689024390243902,
+      "grad_norm": 4.650434494018555,
+      "learning_rate": 4.75e-06,
+      "loss": 0.3189,
+      "step": 95
+    },
+    {
+      "epoch": 0.8780487804878049,
+      "grad_norm": 15.319528579711914,
+      "learning_rate": 4.800000000000001e-06,
+      "loss": 0.3347,
+      "step": 96
+    },
+    {
+      "epoch": 0.8871951219512195,
+      "grad_norm": 5.151925563812256,
+      "learning_rate": 4.85e-06,
+      "loss": 0.3043,
+      "step": 97
+    },
+    {
+      "epoch": 0.8963414634146342,
+      "grad_norm": 6.351178169250488,
+      "learning_rate": 4.9000000000000005e-06,
+      "loss": 0.2939,
+      "step": 98
+    },
+    {
+      "epoch": 0.9054878048780488,
+      "grad_norm": 3.942911148071289,
+      "learning_rate": 4.95e-06,
+      "loss": 0.2887,
+      "step": 99
+    },
+    {
+      "epoch": 0.9146341463414634,
+      "grad_norm": 3.0885493755340576,
+      "learning_rate": 5e-06,
+      "loss": 0.2992,
+      "step": 100
+    },
+    {
+      "epoch": 0.9237804878048781,
+      "grad_norm": 7.023274898529053,
+      "learning_rate": 4.999959803423605e-06,
+      "loss": 0.2607,
+      "step": 101
+    },
+    {
+      "epoch": 0.9329268292682927,
+      "grad_norm": 8.774216651916504,
+      "learning_rate": 4.9998392149870315e-06,
+      "loss": 0.2674,
+      "step": 102
+    },
+    {
+      "epoch": 0.9420731707317073,
+      "grad_norm": 3.5439791679382324,
+      "learning_rate": 4.999638238568072e-06,
+      "loss": 0.2602,
+      "step": 103
+    },
+    {
+      "epoch": 0.9512195121951219,
+      "grad_norm": 7.669381618499756,
+      "learning_rate": 4.999356880629579e-06,
+      "loss": 0.2718,
+      "step": 104
+    },
+    {
+      "epoch": 0.9603658536585366,
+      "grad_norm": 6.9407854080200195,
+      "learning_rate": 4.998995150219252e-06,
+      "loss": 0.2494,
+      "step": 105
+    },
+    {
+      "epoch": 0.9695121951219512,
+      "grad_norm": 12.144411087036133,
+      "learning_rate": 4.998553058969352e-06,
+      "loss": 0.2297,
+      "step": 106
+    },
+    {
+      "epoch": 0.9786585365853658,
+      "grad_norm": 6.956138610839844,
+      "learning_rate": 4.998030621096322e-06,
+      "loss": 0.2386,
+      "step": 107
+    },
+    {
+      "epoch": 0.9878048780487805,
+      "grad_norm": 8.304037094116211,
+      "learning_rate": 4.997427853400333e-06,
+      "loss": 0.23,
+      "step": 108
+    },
+    {
+      "epoch": 0.9969512195121951,
+      "grad_norm": 7.031365394592285,
+      "learning_rate": 4.996744775264743e-06,
+      "loss": 0.2346,
+      "step": 109
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 654,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 6,
+  "save_steps": 109,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 3.308236123527971e+18,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}
diff --git a/checkpoint-109/training_args.bin b/checkpoint-109/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..24b9be216f1966bb6d081a79c30a21dced39cf99
--- /dev/null
+++ b/checkpoint-109/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:14990b8399bb8d39b746405b094f7eea1041d93488de8433b2d415de16ff957a
+size 7928
diff --git a/checkpoint-109/zero_to_fp32.py b/checkpoint-109/zero_to_fp32.py
new file mode 100644
index 0000000000000000000000000000000000000000..24cc342e78d1a006c782b3a4cd68d9ce786d8fd8
--- /dev/null
+++ b/checkpoint-109/zero_to_fp32.py
@@ -0,0 +1,604 @@
+#!/usr/bin/env python
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
+# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
+# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
+# application.
+#
+# example: python zero_to_fp32.py . pytorch_model.bin
+
+import argparse
+import torch
+import glob
+import math
+import os
+import re
+from collections import OrderedDict
+from dataclasses import dataclass
+
+# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
+# DeepSpeed data structures it has to be available in the current python environment.
+from deepspeed.utils import logger
+from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
+                                            FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
+                                            FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS)
+
+
+@dataclass
+class zero_model_state:
+    buffers: dict()
+    param_shapes: dict()
+    shared_params: list
+    ds_version: int
+    frozen_param_shapes: dict()
+    frozen_param_fragments: dict()
+
+
+debug = 0
+
+# load to cpu
+device = torch.device('cpu')
+
+
+def atoi(text):
+    return int(text) if text.isdigit() else text
+
+
+def natural_keys(text):
+    '''
+    alist.sort(key=natural_keys) sorts in human order
+    http://nedbatchelder.com/blog/200712/human_sorting.html
+    (See Toothy's implementation in the comments)
+    '''
+    return [atoi(c) for c in re.split(r'(\d+)', text)]
+
+
+def get_model_state_file(checkpoint_dir, zero_stage):
+    if not os.path.isdir(checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
+
+    # there should be only one file
+    if zero_stage <= 2:
+        file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
+    elif zero_stage == 3:
+        file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
+
+    if not os.path.exists(file):
+        raise FileNotFoundError(f"can't find model states file at '{file}'")
+
+    return file
+
+
+def get_checkpoint_files(checkpoint_dir, glob_pattern):
+    # XXX: need to test that this simple glob rule works for multi-node setup too
+    ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
+
+    if len(ckpt_files) == 0:
+        raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
+
+    return ckpt_files
+
+
+def get_optim_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
+
+
+def get_model_state_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
+
+
+def parse_model_states(files):
+    zero_model_states = []
+    for file in files:
+        state_dict = torch.load(file, map_location=device)
+
+        if BUFFER_NAMES not in state_dict:
+            raise ValueError(f"{file} is not a model state checkpoint")
+        buffer_names = state_dict[BUFFER_NAMES]
+        if debug:
+            print("Found buffers:", buffer_names)
+
+        # recover just the buffers while restoring them to fp32 if they were saved in fp16
+        buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
+        param_shapes = state_dict[PARAM_SHAPES]
+
+        # collect parameters that are included in param_shapes
+        param_names = []
+        for s in param_shapes:
+            for name in s.keys():
+                param_names.append(name)
+
+        # update with frozen parameters
+        frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
+        if frozen_param_shapes is not None:
+            if debug:
+                print(f"Found frozen_param_shapes: {frozen_param_shapes}")
+            param_names += list(frozen_param_shapes.keys())
+
+        # handle shared params
+        shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
+
+        ds_version = state_dict.get(DS_VERSION, None)
+
+        frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
+
+        z_model_state = zero_model_state(buffers=buffers,
+                                         param_shapes=param_shapes,
+                                         shared_params=shared_params,
+                                         ds_version=ds_version,
+                                         frozen_param_shapes=frozen_param_shapes,
+                                         frozen_param_fragments=frozen_param_fragments)
+        zero_model_states.append(z_model_state)
+
+    return zero_model_states
+
+
+def parse_optim_states(files, ds_checkpoint_dir):
+
+    total_files = len(files)
+    state_dicts = []
+    for f in files:
+        state_dict = torch.load(f, map_location=device)
+        # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+        # and also handle the case where it was already removed by another helper script
+        state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+        state_dicts.append(state_dict)
+
+    if not ZERO_STAGE in state_dicts[0][OPTIMIZER_STATE_DICT]:
+        raise ValueError(f"{files[0]} is not a zero checkpoint")
+    zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
+    world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
+
+    # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
+    # parameters can be different from data parallelism for non-expert parameters. So we can just
+    # use the max of the partition_count to get the dp world_size.
+
+    if type(world_size) is list:
+        world_size = max(world_size)
+
+    if world_size != total_files:
+        raise ValueError(
+            f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
+            "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
+        )
+
+    # the groups are named differently in each stage
+    if zero_stage <= 2:
+        fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
+    elif zero_stage == 3:
+        fp32_groups_key = FP32_FLAT_GROUPS
+    else:
+        raise ValueError(f"unknown zero stage {zero_stage}")
+
+    if zero_stage <= 2:
+        fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
+    elif zero_stage == 3:
+        # if there is more than one param group, there will be multiple flattened tensors - one
+        # flattened tensor per group - for simplicity merge them into a single tensor
+        #
+        # XXX: could make the script more memory efficient for when there are multiple groups - it
+        # will require matching the sub-lists of param_shapes for each param group flattened tensor
+
+        fp32_flat_groups = [
+            torch.cat(state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key], 0) for i in range(len(state_dicts))
+        ]
+
+    return zero_stage, world_size, fp32_flat_groups
+
+
+def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
+    """
+    Returns fp32 state_dict reconstructed from ds checkpoint
+
+    Args:
+        - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
+
+    """
+    print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
+
+    optim_files = get_optim_files(ds_checkpoint_dir)
+    zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
+    print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
+
+    model_files = get_model_state_files(ds_checkpoint_dir)
+
+    zero_model_states = parse_model_states(model_files)
+    print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
+
+    if zero_stage <= 2:
+        return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+    elif zero_stage == 3:
+        return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+
+
+def _zero2_merge_frozen_params(state_dict, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+    frozen_param_fragments = zero_model_states[0].frozen_param_fragments
+
+    if debug:
+        num_elem = sum(s.numel() for s in frozen_param_shapes.values())
+        print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        state_dict[name] = frozen_param_fragments[name]
+
+        if debug:
+            print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _has_callable(obj, fn):
+    attr = getattr(obj, fn, None)
+    return callable(attr)
+
+
+def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+
+    # Reconstruction protocol:
+    #
+    # XXX: document this
+
+    if debug:
+        for i in range(world_size):
+            for j in range(len(fp32_flat_groups[0])):
+                print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
+
+    # XXX: memory usage doubles here (zero2)
+    num_param_groups = len(fp32_flat_groups[0])
+    merged_single_partition_of_fp32_groups = []
+    for i in range(num_param_groups):
+        merged_partitions = [sd[i] for sd in fp32_flat_groups]
+        full_single_fp32_vector = torch.cat(merged_partitions, 0)
+        merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
+    avail_numel = sum(
+        [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
+
+    if debug:
+        wanted_params = sum([len(shapes) for shapes in param_shapes])
+        wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
+        # not asserting if there is a mismatch due to possible padding
+        print(f"Have {avail_numel} numels to process.")
+        print(f"Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    total_numel = 0
+    total_params = 0
+    for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
+        offset = 0
+        avail_numel = full_single_fp32_vector.numel()
+        for name, shape in shapes.items():
+
+            unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
+            total_numel += unpartitioned_numel
+            total_params += 1
+
+            if debug:
+                print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+            state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
+            offset += unpartitioned_numel
+
+        # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
+        # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
+        # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
+        # live optimizer object, so we are checking that the numbers are within the right range
+        align_to = 2 * world_size
+
+        def zero2_align(x):
+            return align_to * math.ceil(x / align_to)
+
+        if debug:
+            print(f"original offset={offset}, avail_numel={avail_numel}")
+
+        offset = zero2_align(offset)
+        avail_numel = zero2_align(avail_numel)
+
+        if debug:
+            print(f"aligned  offset={offset}, avail_numel={avail_numel}")
+
+        # Sanity check
+        if offset != avail_numel:
+            raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero2_merge_frozen_params(state_dict, zero_model_states)
+
+    _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def zero3_partitioned_param_info(unpartitioned_numel, world_size):
+    remainder = unpartitioned_numel % world_size
+    padding_numel = (world_size - remainder) if remainder else 0
+    partitioned_numel = math.ceil(unpartitioned_numel / world_size)
+    return partitioned_numel, padding_numel
+
+
+def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    if debug:
+        for i in range(world_size):
+            num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
+            print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in zero_model_states[0].frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
+        state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
+
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+    avail_numel = fp32_flat_groups[0].numel() * world_size
+    # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
+    # param, re-consolidating each param, while dealing with padding if any
+
+    # merge list of dicts, preserving order
+    param_shapes = {k: v for d in param_shapes for k, v in d.items()}
+
+    if debug:
+        for i in range(world_size):
+            print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
+
+        wanted_params = len(param_shapes)
+        wanted_numel = sum(shape.numel() for shape in param_shapes.values())
+        # not asserting if there is a mismatch due to possible padding
+        avail_numel = fp32_flat_groups[0].numel() * world_size
+        print(f"Trainable params: Have {avail_numel} numels to process.")
+        print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    offset = 0
+    total_numel = 0
+    total_params = 0
+    for name, shape in param_shapes.items():
+
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+        total_params += 1
+
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+        # XXX: memory usage doubles here
+        state_dict[name] = torch.cat(
+            tuple(fp32_flat_groups[i].narrow(0, offset, partitioned_numel) for i in range(world_size)),
+            0).narrow(0, 0, unpartitioned_numel).view(shape)
+        offset += partitioned_numel
+
+    offset *= world_size
+
+    # Sanity check
+    if offset != avail_numel:
+        raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
+
+    _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag=None, exclude_frozen_parameters=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
+    ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
+    via a model hub.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+
+    Returns:
+        - pytorch ``state_dict``
+
+    Note: this approach may not work if your application doesn't have sufficient free CPU memory and
+    you may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
+    the checkpoint.
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+        # do the training and checkpoint saving
+        state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
+        model = model.cpu() # move to cpu
+        model.load_state_dict(state_dict)
+        # submit to model hub or save the model to share with others
+
+    In this example the ``model`` will no longer be usable in the deepspeed context of the same
+    application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
+
+    """
+    if tag is None:
+        latest_path = os.path.join(checkpoint_dir, 'latest')
+        if os.path.isfile(latest_path):
+            with open(latest_path, 'r') as fd:
+                tag = fd.read().strip()
+        else:
+            raise ValueError(f"Unable to find 'latest' file at {latest_path}")
+
+    ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
+
+    if not os.path.isdir(ds_checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
+
+    return _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
+
+
+def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, output_file, tag=None, exclude_frozen_parameters=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
+    loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``output_file``: path to the pytorch fp32 state_dict output file (e.g. path/pytorch_model.bin)
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+    """
+
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag, exclude_frozen_parameters)
+    print(f"Saving fp32 state dict to {output_file}")
+    torch.save(state_dict, output_file)
+
+
+def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
+    """
+    1. Put the provided model to cpu
+    2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
+    3. Load it into the provided model
+
+    Args:
+        - ``model``: the model object to update
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+
+    Returns:
+        - ``model`: modified model
+
+    Make sure you have plenty of CPU memory available before you call this function. If you don't
+    have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
+    conveniently placed for you in the checkpoint folder.
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
+        model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
+        # submit to model hub or save the model to share with others
+
+    Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
+    of the same application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    """
+    logger.info(f"Extracting fp32 weights")
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+
+    logger.info(f"Overwriting model with fp32 weights")
+    model = model.cpu()
+    model.load_state_dict(state_dict, strict=False)
+
+    return model
+
+
+if __name__ == "__main__":
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("checkpoint_dir",
+                        type=str,
+                        help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+    parser.add_argument(
+        "output_file",
+        type=str,
+        help="path to the pytorch fp32 state_dict output file (e.g. path/checkpoint-12/pytorch_model.bin)")
+    parser.add_argument("-t",
+                        "--tag",
+                        type=str,
+                        default=None,
+                        help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
+    parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+    parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
+    args = parser.parse_args()
+
+    debug = args.debug
+
+    convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
+                                               args.output_file,
+                                               tag=args.tag,
+                                               exclude_frozen_parameters=args.exclude_frozen_parameters)
diff --git a/checkpoint-218/README.md b/checkpoint-218/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..f4a3934800eeb082a0cb833d7b6af4f68eed3615
--- /dev/null
+++ b/checkpoint-218/README.md
@@ -0,0 +1,202 @@
+---
+base_model: nvidia/Llama-3_3-Nemotron-Super-49B-v1
+library_name: peft
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.15.0
\ No newline at end of file
diff --git a/checkpoint-218/adapter_config.json b/checkpoint-218/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..d7d7c196589a89c5b284ba682f27212262154f44
--- /dev/null
+++ b/checkpoint-218/adapter_config.json
@@ -0,0 +1,42 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "nvidia/Llama-3_3-Nemotron-Super-49B-v1",
+  "bias": "none",
+  "corda_config": null,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": null,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 512,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": [
+    "embed_tokens",
+    "lm_head"
+  ],
+  "peft_type": "LORA",
+  "r": 256,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "o_proj",
+    "gate_proj",
+    "q_proj",
+    "k_proj",
+    "v_proj",
+    "up_proj",
+    "down_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_rslora": false
+}
\ No newline at end of file
diff --git a/checkpoint-218/adapter_model.safetensors b/checkpoint-218/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..1fa0cbdc985ea654dda8ae2753a71fd54b254046
--- /dev/null
+++ b/checkpoint-218/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b76fdaabeb0342548ab74dece3dcaef2498cd01d11ae6496e161b50f85d39886
+size 9016826528
diff --git a/checkpoint-218/global_step217/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/checkpoint-218/global_step217/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4d8fcece901189b64e99bcb13b87fac9f69b6660
--- /dev/null
+++ b/checkpoint-218/global_step217/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cb254d842a0bd3ac4dc30619d358c802076f2ae5a8a1d7b220ef274adcb63e50
+size 27050164444
diff --git a/checkpoint-218/global_step217/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/checkpoint-218/global_step217/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..60b38890cb8cd660d582040f97271a532e99923d
--- /dev/null
+++ b/checkpoint-218/global_step217/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1c224fcc95287ab5f4cbea9ba6ab0ab3f61c93b93fe2482cba1cfcbbc438f1bb
+size 27050169884
diff --git a/checkpoint-218/global_step217/mp_rank_00_model_states.pt b/checkpoint-218/global_step217/mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2b3edb796056c80e1f44ae7e207b7fb87901ef88
--- /dev/null
+++ b/checkpoint-218/global_step217/mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:90a15492216e48ac85875cb87b3ab32a526439c0341ca98b34dc9ae767d8123e
+size 9776788601
diff --git a/checkpoint-218/latest b/checkpoint-218/latest
new file mode 100644
index 0000000000000000000000000000000000000000..69efd65b61133c537b6eac06019baa6ef251019d
--- /dev/null
+++ b/checkpoint-218/latest
@@ -0,0 +1 @@
+global_step217
\ No newline at end of file
diff --git a/checkpoint-218/rng_state_0.pth b/checkpoint-218/rng_state_0.pth
new file mode 100644
index 0000000000000000000000000000000000000000..18e27483b9d240fc01f00d642c6f774a77d27a08
--- /dev/null
+++ b/checkpoint-218/rng_state_0.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:15e06bff2ae5de048d5791e7d5b33c487c72cb4c6226bbe98529b686c66d1960
+size 14512
diff --git a/checkpoint-218/rng_state_1.pth b/checkpoint-218/rng_state_1.pth
new file mode 100644
index 0000000000000000000000000000000000000000..7565aee35482ded95aa201ab1d0f0dc5f82479d7
--- /dev/null
+++ b/checkpoint-218/rng_state_1.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9ebf3b1d63337eb96a8c9ec5fb78befa637a5ae4dce16a50b68163cfabe6dd02
+size 14512
diff --git a/checkpoint-218/scheduler.pt b/checkpoint-218/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9d89fa09d4ea88afcd00b80800516bb379e34a5e
--- /dev/null
+++ b/checkpoint-218/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:33fe6cc8ba4774c5504d871fd373473a25f477a0e4a445a512e0595aa3abfbab
+size 1064
diff --git a/checkpoint-218/special_tokens_map.json b/checkpoint-218/special_tokens_map.json
new file mode 100644
index 0000000000000000000000000000000000000000..278b7f0f84be865c4687700ee7b3c63d89a51e18
--- /dev/null
+++ b/checkpoint-218/special_tokens_map.json
@@ -0,0 +1,23 @@
+{
+  "bos_token": {
+    "content": "<|begin_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|eot_id|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|end_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}
diff --git a/checkpoint-218/tokenizer.json b/checkpoint-218/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..1c1d8d5c9024994f1d3b00f9662b8dd89ca13cf2
--- /dev/null
+++ b/checkpoint-218/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920
diff --git a/checkpoint-218/tokenizer_config.json b/checkpoint-218/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..edd01b980c1db496ea102a51c972ee8f5d1a2c74
--- /dev/null
+++ b/checkpoint-218/tokenizer_config.json
@@ -0,0 +1,2064 @@
+{
+  "added_tokens_decoder": {
+    "128000": {
+      "content": "<|begin_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128001": {
+      "content": "<|end_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128002": {
+      "content": "<|reserved_special_token_0|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128003": {
+      "content": "<|reserved_special_token_1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128004": {
+      "content": "<|finetune_right_pad_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128005": {
+      "content": "<|reserved_special_token_2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128006": {
+      "content": "<|start_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128007": {
+      "content": "<|end_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128008": {
+      "content": "<|eom_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128009": {
+      "content": "<|eot_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128010": {
+      "content": "<|python_tag|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128011": {
+      "content": "<|reserved_special_token_3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128012": {
+      "content": "<|reserved_special_token_4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128013": {
+      "content": "<|reserved_special_token_5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128014": {
+      "content": "<|reserved_special_token_6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128015": {
+      "content": "<|reserved_special_token_7|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128016": {
+      "content": "<|reserved_special_token_8|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128017": {
+      "content": "<|reserved_special_token_9|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128018": {
+      "content": "<|reserved_special_token_10|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128019": {
+      "content": "<|reserved_special_token_11|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128020": {
+      "content": "<|reserved_special_token_12|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128021": {
+      "content": "<|reserved_special_token_13|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128022": {
+      "content": "<|reserved_special_token_14|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128023": {
+      "content": "<|reserved_special_token_15|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128024": {
+      "content": "<|reserved_special_token_16|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128025": {
+      "content": "<|reserved_special_token_17|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128026": {
+      "content": "<|reserved_special_token_18|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128027": {
+      "content": "<|reserved_special_token_19|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128028": {
+      "content": "<|reserved_special_token_20|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128029": {
+      "content": "<|reserved_special_token_21|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128030": {
+      "content": "<|reserved_special_token_22|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128031": {
+      "content": "<|reserved_special_token_23|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128032": {
+      "content": "<|reserved_special_token_24|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128033": {
+      "content": "<|reserved_special_token_25|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128034": {
+      "content": "<|reserved_special_token_26|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128035": {
+      "content": "<|reserved_special_token_27|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128036": {
+      "content": "<|reserved_special_token_28|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128037": {
+      "content": "<|reserved_special_token_29|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128038": {
+      "content": "<|reserved_special_token_30|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128039": {
+      "content": "<|reserved_special_token_31|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128040": {
+      "content": "<|reserved_special_token_32|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128041": {
+      "content": "<|reserved_special_token_33|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128042": {
+      "content": "<|reserved_special_token_34|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128043": {
+      "content": "<|reserved_special_token_35|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128044": {
+      "content": "<|reserved_special_token_36|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128045": {
+      "content": "<|reserved_special_token_37|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128046": {
+      "content": "<|reserved_special_token_38|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128047": {
+      "content": "<|reserved_special_token_39|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128048": {
+      "content": "<|reserved_special_token_40|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128049": {
+      "content": "<|reserved_special_token_41|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128050": {
+      "content": "<|reserved_special_token_42|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128051": {
+      "content": "<|reserved_special_token_43|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128052": {
+      "content": "<|reserved_special_token_44|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128053": {
+      "content": "<|reserved_special_token_45|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128054": {
+      "content": "<|reserved_special_token_46|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128055": {
+      "content": "<|reserved_special_token_47|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128056": {
+      "content": "<|reserved_special_token_48|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128057": {
+      "content": "<|reserved_special_token_49|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128058": {
+      "content": "<|reserved_special_token_50|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128059": {
+      "content": "<|reserved_special_token_51|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128060": {
+      "content": "<|reserved_special_token_52|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128061": {
+      "content": "<|reserved_special_token_53|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128062": {
+      "content": "<|reserved_special_token_54|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128063": {
+      "content": "<|reserved_special_token_55|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128064": {
+      "content": "<|reserved_special_token_56|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128065": {
+      "content": "<|reserved_special_token_57|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128066": {
+      "content": "<|reserved_special_token_58|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128067": {
+      "content": "<|reserved_special_token_59|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128068": {
+      "content": "<|reserved_special_token_60|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128069": {
+      "content": "<|reserved_special_token_61|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128070": {
+      "content": "<|reserved_special_token_62|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128071": {
+      "content": "<|reserved_special_token_63|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128072": {
+      "content": "<|reserved_special_token_64|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128073": {
+      "content": "<|reserved_special_token_65|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128074": {
+      "content": "<|reserved_special_token_66|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128075": {
+      "content": "<|reserved_special_token_67|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128076": {
+      "content": "<|reserved_special_token_68|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128077": {
+      "content": "<|reserved_special_token_69|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128078": {
+      "content": "<|reserved_special_token_70|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128079": {
+      "content": "<|reserved_special_token_71|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128080": {
+      "content": "<|reserved_special_token_72|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128081": {
+      "content": "<|reserved_special_token_73|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128082": {
+      "content": "<|reserved_special_token_74|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128083": {
+      "content": "<|reserved_special_token_75|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128084": {
+      "content": "<|reserved_special_token_76|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128085": {
+      "content": "<|reserved_special_token_77|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128086": {
+      "content": "<|reserved_special_token_78|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128087": {
+      "content": "<|reserved_special_token_79|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128088": {
+      "content": "<|reserved_special_token_80|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128089": {
+      "content": "<|reserved_special_token_81|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128090": {
+      "content": "<|reserved_special_token_82|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128091": {
+      "content": "<|reserved_special_token_83|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128092": {
+      "content": "<|reserved_special_token_84|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128093": {
+      "content": "<|reserved_special_token_85|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128094": {
+      "content": "<|reserved_special_token_86|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128095": {
+      "content": "<|reserved_special_token_87|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128096": {
+      "content": "<|reserved_special_token_88|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128097": {
+      "content": "<|reserved_special_token_89|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128098": {
+      "content": "<|reserved_special_token_90|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128099": {
+      "content": "<|reserved_special_token_91|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128100": {
+      "content": "<|reserved_special_token_92|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128101": {
+      "content": "<|reserved_special_token_93|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128102": {
+      "content": "<|reserved_special_token_94|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128103": {
+      "content": "<|reserved_special_token_95|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128104": {
+      "content": "<|reserved_special_token_96|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128105": {
+      "content": "<|reserved_special_token_97|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128106": {
+      "content": "<|reserved_special_token_98|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128107": {
+      "content": "<|reserved_special_token_99|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128108": {
+      "content": "<|reserved_special_token_100|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128109": {
+      "content": "<|reserved_special_token_101|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128110": {
+      "content": "<|reserved_special_token_102|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128111": {
+      "content": "<|reserved_special_token_103|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128112": {
+      "content": "<|reserved_special_token_104|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128113": {
+      "content": "<|reserved_special_token_105|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128114": {
+      "content": "<|reserved_special_token_106|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128115": {
+      "content": "<|reserved_special_token_107|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128116": {
+      "content": "<|reserved_special_token_108|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128117": {
+      "content": "<|reserved_special_token_109|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128118": {
+      "content": "<|reserved_special_token_110|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128119": {
+      "content": "<|reserved_special_token_111|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128120": {
+      "content": "<|reserved_special_token_112|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128121": {
+      "content": "<|reserved_special_token_113|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128122": {
+      "content": "<|reserved_special_token_114|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128123": {
+      "content": "<|reserved_special_token_115|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128124": {
+      "content": "<|reserved_special_token_116|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128125": {
+      "content": "<|reserved_special_token_117|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128126": {
+      "content": "<|reserved_special_token_118|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128127": {
+      "content": "<|reserved_special_token_119|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128128": {
+      "content": "<|reserved_special_token_120|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128129": {
+      "content": "<|reserved_special_token_121|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128130": {
+      "content": "<|reserved_special_token_122|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128131": {
+      "content": "<|reserved_special_token_123|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128132": {
+      "content": "<|reserved_special_token_124|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128133": {
+      "content": "<|reserved_special_token_125|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128134": {
+      "content": "<|reserved_special_token_126|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128135": {
+      "content": "<|reserved_special_token_127|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128136": {
+      "content": "<|reserved_special_token_128|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128137": {
+      "content": "<|reserved_special_token_129|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128138": {
+      "content": "<|reserved_special_token_130|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128139": {
+      "content": "<|reserved_special_token_131|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128140": {
+      "content": "<|reserved_special_token_132|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128141": {
+      "content": "<|reserved_special_token_133|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128142": {
+      "content": "<|reserved_special_token_134|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128143": {
+      "content": "<|reserved_special_token_135|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128144": {
+      "content": "<|reserved_special_token_136|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128145": {
+      "content": "<|reserved_special_token_137|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128146": {
+      "content": "<|reserved_special_token_138|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128147": {
+      "content": "<|reserved_special_token_139|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128148": {
+      "content": "<|reserved_special_token_140|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128149": {
+      "content": "<|reserved_special_token_141|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128150": {
+      "content": "<|reserved_special_token_142|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128151": {
+      "content": "<|reserved_special_token_143|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128152": {
+      "content": "<|reserved_special_token_144|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128153": {
+      "content": "<|reserved_special_token_145|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128154": {
+      "content": "<|reserved_special_token_146|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128155": {
+      "content": "<|reserved_special_token_147|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128156": {
+      "content": "<|reserved_special_token_148|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128157": {
+      "content": "<|reserved_special_token_149|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128158": {
+      "content": "<|reserved_special_token_150|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128159": {
+      "content": "<|reserved_special_token_151|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128160": {
+      "content": "<|reserved_special_token_152|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128161": {
+      "content": "<|reserved_special_token_153|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128162": {
+      "content": "<|reserved_special_token_154|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128163": {
+      "content": "<|reserved_special_token_155|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128164": {
+      "content": "<|reserved_special_token_156|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128165": {
+      "content": "<|reserved_special_token_157|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128166": {
+      "content": "<|reserved_special_token_158|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128167": {
+      "content": "<|reserved_special_token_159|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128168": {
+      "content": "<|reserved_special_token_160|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128169": {
+      "content": "<|reserved_special_token_161|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128170": {
+      "content": "<|reserved_special_token_162|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128171": {
+      "content": "<|reserved_special_token_163|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128172": {
+      "content": "<|reserved_special_token_164|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128173": {
+      "content": "<|reserved_special_token_165|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128174": {
+      "content": "<|reserved_special_token_166|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128175": {
+      "content": "<|reserved_special_token_167|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128176": {
+      "content": "<|reserved_special_token_168|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128177": {
+      "content": "<|reserved_special_token_169|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128178": {
+      "content": "<|reserved_special_token_170|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128179": {
+      "content": "<|reserved_special_token_171|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128180": {
+      "content": "<|reserved_special_token_172|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128181": {
+      "content": "<|reserved_special_token_173|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128182": {
+      "content": "<|reserved_special_token_174|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128183": {
+      "content": "<|reserved_special_token_175|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128184": {
+      "content": "<|reserved_special_token_176|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128185": {
+      "content": "<|reserved_special_token_177|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128186": {
+      "content": "<|reserved_special_token_178|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128187": {
+      "content": "<|reserved_special_token_179|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128188": {
+      "content": "<|reserved_special_token_180|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128189": {
+      "content": "<|reserved_special_token_181|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128190": {
+      "content": "<|reserved_special_token_182|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128191": {
+      "content": "<|reserved_special_token_183|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128192": {
+      "content": "<|reserved_special_token_184|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128193": {
+      "content": "<|reserved_special_token_185|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128194": {
+      "content": "<|reserved_special_token_186|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128195": {
+      "content": "<|reserved_special_token_187|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128196": {
+      "content": "<|reserved_special_token_188|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128197": {
+      "content": "<|reserved_special_token_189|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128198": {
+      "content": "<|reserved_special_token_190|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128199": {
+      "content": "<|reserved_special_token_191|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128200": {
+      "content": "<|reserved_special_token_192|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128201": {
+      "content": "<|reserved_special_token_193|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128202": {
+      "content": "<|reserved_special_token_194|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128203": {
+      "content": "<|reserved_special_token_195|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128204": {
+      "content": "<|reserved_special_token_196|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128205": {
+      "content": "<|reserved_special_token_197|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128206": {
+      "content": "<|reserved_special_token_198|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128207": {
+      "content": "<|reserved_special_token_199|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128208": {
+      "content": "<|reserved_special_token_200|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128209": {
+      "content": "<|reserved_special_token_201|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128210": {
+      "content": "<|reserved_special_token_202|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128211": {
+      "content": "<|reserved_special_token_203|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128212": {
+      "content": "<|reserved_special_token_204|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128213": {
+      "content": "<|reserved_special_token_205|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128214": {
+      "content": "<|reserved_special_token_206|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128215": {
+      "content": "<|reserved_special_token_207|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128216": {
+      "content": "<|reserved_special_token_208|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128217": {
+      "content": "<|reserved_special_token_209|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128218": {
+      "content": "<|reserved_special_token_210|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128219": {
+      "content": "<|reserved_special_token_211|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128220": {
+      "content": "<|reserved_special_token_212|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128221": {
+      "content": "<|reserved_special_token_213|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128222": {
+      "content": "<|reserved_special_token_214|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128223": {
+      "content": "<|reserved_special_token_215|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128224": {
+      "content": "<|reserved_special_token_216|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128225": {
+      "content": "<|reserved_special_token_217|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128226": {
+      "content": "<|reserved_special_token_218|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128227": {
+      "content": "<|reserved_special_token_219|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128228": {
+      "content": "<|reserved_special_token_220|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128229": {
+      "content": "<|reserved_special_token_221|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128230": {
+      "content": "<|reserved_special_token_222|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128231": {
+      "content": "<|reserved_special_token_223|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128232": {
+      "content": "<|reserved_special_token_224|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128233": {
+      "content": "<|reserved_special_token_225|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128234": {
+      "content": "<|reserved_special_token_226|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128235": {
+      "content": "<|reserved_special_token_227|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128236": {
+      "content": "<|reserved_special_token_228|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128237": {
+      "content": "<|reserved_special_token_229|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128238": {
+      "content": "<|reserved_special_token_230|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128239": {
+      "content": "<|reserved_special_token_231|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128240": {
+      "content": "<|reserved_special_token_232|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128241": {
+      "content": "<|reserved_special_token_233|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128242": {
+      "content": "<|reserved_special_token_234|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128243": {
+      "content": "<|reserved_special_token_235|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128244": {
+      "content": "<|reserved_special_token_236|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128245": {
+      "content": "<|reserved_special_token_237|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128246": {
+      "content": "<|reserved_special_token_238|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128247": {
+      "content": "<|reserved_special_token_239|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128248": {
+      "content": "<|reserved_special_token_240|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128249": {
+      "content": "<|reserved_special_token_241|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128250": {
+      "content": "<|reserved_special_token_242|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128251": {
+      "content": "<|reserved_special_token_243|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128252": {
+      "content": "<|reserved_special_token_244|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128253": {
+      "content": "<|reserved_special_token_245|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128254": {
+      "content": "<|reserved_special_token_246|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128255": {
+      "content": "<|reserved_special_token_247|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|begin_of_text|>",
+  "chat_template": "{{- bos_token }}{%- if messages[0]['role'] == 'system' %}{%- set system_message = messages[0]['content']|trim %}{%- set messages = messages[1:] %}{%- else %}{%- set system_message = \"\" %}{%- endif %}{{- \"<|start_header_id|>system<|end_header_id|>\\n\\n\" }}{{- system_message }}{{- \"<|eot_id|>\" }}{%- for message in messages %}{%- if message['role'] == 'assistant' and '</think>' in message['content'] %}{%- set content = message['content'].split('</think>')[-1].lstrip() %}{%- else %}{%- set content = message['content'] %}{%- endif %}{{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\\n\\n' + content | trim + '<|eot_id|>' }}{%- endfor %}{%- if add_generation_prompt %}{{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' }}{%- endif %}",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|eot_id|>",
+  "extra_special_tokens": {},
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "pad_token": "<|end_of_text|>",
+  "tokenizer_class": "PreTrainedTokenizer"
+}
diff --git a/checkpoint-218/trainer_state.json b/checkpoint-218/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..e201a9ce48af6fd40cefc96b3044a57866fe91a1
--- /dev/null
+++ b/checkpoint-218/trainer_state.json
@@ -0,0 +1,1559 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.9878048780487805,
+  "eval_steps": 500,
+  "global_step": 218,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.009146341463414634,
+      "grad_norm": 40.05127716064453,
+      "learning_rate": 5.0000000000000004e-08,
+      "loss": 5.1282,
+      "step": 1
+    },
+    {
+      "epoch": 0.018292682926829267,
+      "grad_norm": 39.01039505004883,
+      "learning_rate": 1.0000000000000001e-07,
+      "loss": 5.0586,
+      "step": 2
+    },
+    {
+      "epoch": 0.027439024390243903,
+      "grad_norm": 40.20766067504883,
+      "learning_rate": 1.5000000000000002e-07,
+      "loss": 5.0311,
+      "step": 3
+    },
+    {
+      "epoch": 0.036585365853658534,
+      "grad_norm": 38.70350646972656,
+      "learning_rate": 2.0000000000000002e-07,
+      "loss": 5.1598,
+      "step": 4
+    },
+    {
+      "epoch": 0.04573170731707317,
+      "grad_norm": 39.230979919433594,
+      "learning_rate": 2.5000000000000004e-07,
+      "loss": 5.0812,
+      "step": 5
+    },
+    {
+      "epoch": 0.054878048780487805,
+      "grad_norm": 39.45293045043945,
+      "learning_rate": 3.0000000000000004e-07,
+      "loss": 5.0841,
+      "step": 6
+    },
+    {
+      "epoch": 0.06402439024390244,
+      "grad_norm": 38.92170333862305,
+      "learning_rate": 3.5000000000000004e-07,
+      "loss": 5.022,
+      "step": 7
+    },
+    {
+      "epoch": 0.07317073170731707,
+      "grad_norm": 38.932369232177734,
+      "learning_rate": 4.0000000000000003e-07,
+      "loss": 5.0876,
+      "step": 8
+    },
+    {
+      "epoch": 0.08231707317073171,
+      "grad_norm": 39.60245132446289,
+      "learning_rate": 4.5000000000000003e-07,
+      "loss": 5.0798,
+      "step": 9
+    },
+    {
+      "epoch": 0.09146341463414634,
+      "grad_norm": 39.53607177734375,
+      "learning_rate": 5.000000000000001e-07,
+      "loss": 5.0105,
+      "step": 10
+    },
+    {
+      "epoch": 0.10060975609756098,
+      "grad_norm": 39.50700378417969,
+      "learning_rate": 5.5e-07,
+      "loss": 4.9951,
+      "step": 11
+    },
+    {
+      "epoch": 0.10975609756097561,
+      "grad_norm": 39.52376174926758,
+      "learning_rate": 6.000000000000001e-07,
+      "loss": 4.9163,
+      "step": 12
+    },
+    {
+      "epoch": 0.11890243902439024,
+      "grad_norm": 39.15214157104492,
+      "learning_rate": 6.5e-07,
+      "loss": 4.9634,
+      "step": 13
+    },
+    {
+      "epoch": 0.12804878048780488,
+      "grad_norm": 37.795066833496094,
+      "learning_rate": 7.000000000000001e-07,
+      "loss": 4.819,
+      "step": 14
+    },
+    {
+      "epoch": 0.13719512195121952,
+      "grad_norm": 38.23014450073242,
+      "learning_rate": 7.5e-07,
+      "loss": 4.8756,
+      "step": 15
+    },
+    {
+      "epoch": 0.14634146341463414,
+      "grad_norm": 38.57551574707031,
+      "learning_rate": 8.000000000000001e-07,
+      "loss": 4.81,
+      "step": 16
+    },
+    {
+      "epoch": 0.15548780487804878,
+      "grad_norm": 36.06121063232422,
+      "learning_rate": 8.500000000000001e-07,
+      "loss": 4.7901,
+      "step": 17
+    },
+    {
+      "epoch": 0.16463414634146342,
+      "grad_norm": 34.38008117675781,
+      "learning_rate": 9.000000000000001e-07,
+      "loss": 4.5775,
+      "step": 18
+    },
+    {
+      "epoch": 0.17378048780487804,
+      "grad_norm": 32.60890579223633,
+      "learning_rate": 9.500000000000001e-07,
+      "loss": 4.3737,
+      "step": 19
+    },
+    {
+      "epoch": 0.18292682926829268,
+      "grad_norm": 31.983110427856445,
+      "learning_rate": 1.0000000000000002e-06,
+      "loss": 4.3773,
+      "step": 20
+    },
+    {
+      "epoch": 0.19207317073170732,
+      "grad_norm": 27.69571876525879,
+      "learning_rate": 1.0500000000000001e-06,
+      "loss": 4.1276,
+      "step": 21
+    },
+    {
+      "epoch": 0.20121951219512196,
+      "grad_norm": 26.542314529418945,
+      "learning_rate": 1.1e-06,
+      "loss": 4.0641,
+      "step": 22
+    },
+    {
+      "epoch": 0.21036585365853658,
+      "grad_norm": 23.26603126525879,
+      "learning_rate": 1.1500000000000002e-06,
+      "loss": 4.0197,
+      "step": 23
+    },
+    {
+      "epoch": 0.21951219512195122,
+      "grad_norm": 21.04385757446289,
+      "learning_rate": 1.2000000000000002e-06,
+      "loss": 4.0179,
+      "step": 24
+    },
+    {
+      "epoch": 0.22865853658536586,
+      "grad_norm": 19.790103912353516,
+      "learning_rate": 1.25e-06,
+      "loss": 3.8334,
+      "step": 25
+    },
+    {
+      "epoch": 0.23780487804878048,
+      "grad_norm": 18.2396183013916,
+      "learning_rate": 1.3e-06,
+      "loss": 3.7027,
+      "step": 26
+    },
+    {
+      "epoch": 0.24695121951219512,
+      "grad_norm": 18.725860595703125,
+      "learning_rate": 1.3500000000000002e-06,
+      "loss": 3.7268,
+      "step": 27
+    },
+    {
+      "epoch": 0.25609756097560976,
+      "grad_norm": 18.91561508178711,
+      "learning_rate": 1.4000000000000001e-06,
+      "loss": 3.52,
+      "step": 28
+    },
+    {
+      "epoch": 0.2652439024390244,
+      "grad_norm": 18.196651458740234,
+      "learning_rate": 1.45e-06,
+      "loss": 3.4765,
+      "step": 29
+    },
+    {
+      "epoch": 0.27439024390243905,
+      "grad_norm": 17.778606414794922,
+      "learning_rate": 1.5e-06,
+      "loss": 3.3573,
+      "step": 30
+    },
+    {
+      "epoch": 0.28353658536585363,
+      "grad_norm": 16.23012924194336,
+      "learning_rate": 1.5500000000000002e-06,
+      "loss": 3.2783,
+      "step": 31
+    },
+    {
+      "epoch": 0.2926829268292683,
+      "grad_norm": 15.836565017700195,
+      "learning_rate": 1.6000000000000001e-06,
+      "loss": 3.2319,
+      "step": 32
+    },
+    {
+      "epoch": 0.3018292682926829,
+      "grad_norm": 11.919289588928223,
+      "learning_rate": 1.6500000000000003e-06,
+      "loss": 3.1345,
+      "step": 33
+    },
+    {
+      "epoch": 0.31097560975609756,
+      "grad_norm": 11.086843490600586,
+      "learning_rate": 1.7000000000000002e-06,
+      "loss": 3.1153,
+      "step": 34
+    },
+    {
+      "epoch": 0.3201219512195122,
+      "grad_norm": 7.563486576080322,
+      "learning_rate": 1.75e-06,
+      "loss": 2.9784,
+      "step": 35
+    },
+    {
+      "epoch": 0.32926829268292684,
+      "grad_norm": 7.60545539855957,
+      "learning_rate": 1.8000000000000001e-06,
+      "loss": 2.8568,
+      "step": 36
+    },
+    {
+      "epoch": 0.3384146341463415,
+      "grad_norm": 7.071218013763428,
+      "learning_rate": 1.85e-06,
+      "loss": 2.8879,
+      "step": 37
+    },
+    {
+      "epoch": 0.3475609756097561,
+      "grad_norm": 8.200672149658203,
+      "learning_rate": 1.9000000000000002e-06,
+      "loss": 2.7437,
+      "step": 38
+    },
+    {
+      "epoch": 0.3567073170731707,
+      "grad_norm": 12.079374313354492,
+      "learning_rate": 1.9500000000000004e-06,
+      "loss": 2.7589,
+      "step": 39
+    },
+    {
+      "epoch": 0.36585365853658536,
+      "grad_norm": 12.21938705444336,
+      "learning_rate": 2.0000000000000003e-06,
+      "loss": 2.7098,
+      "step": 40
+    },
+    {
+      "epoch": 0.375,
+      "grad_norm": 10.224228858947754,
+      "learning_rate": 2.05e-06,
+      "loss": 2.6803,
+      "step": 41
+    },
+    {
+      "epoch": 0.38414634146341464,
+      "grad_norm": 10.054616928100586,
+      "learning_rate": 2.1000000000000002e-06,
+      "loss": 2.4658,
+      "step": 42
+    },
+    {
+      "epoch": 0.3932926829268293,
+      "grad_norm": 8.074049949645996,
+      "learning_rate": 2.15e-06,
+      "loss": 2.6341,
+      "step": 43
+    },
+    {
+      "epoch": 0.4024390243902439,
+      "grad_norm": 5.721987724304199,
+      "learning_rate": 2.2e-06,
+      "loss": 2.3848,
+      "step": 44
+    },
+    {
+      "epoch": 0.4115853658536585,
+      "grad_norm": 4.646566390991211,
+      "learning_rate": 2.25e-06,
+      "loss": 2.4323,
+      "step": 45
+    },
+    {
+      "epoch": 0.42073170731707316,
+      "grad_norm": 4.591104984283447,
+      "learning_rate": 2.3000000000000004e-06,
+      "loss": 2.4022,
+      "step": 46
+    },
+    {
+      "epoch": 0.4298780487804878,
+      "grad_norm": 4.822715759277344,
+      "learning_rate": 2.35e-06,
+      "loss": 2.2768,
+      "step": 47
+    },
+    {
+      "epoch": 0.43902439024390244,
+      "grad_norm": 5.176568984985352,
+      "learning_rate": 2.4000000000000003e-06,
+      "loss": 2.281,
+      "step": 48
+    },
+    {
+      "epoch": 0.4481707317073171,
+      "grad_norm": 4.731242656707764,
+      "learning_rate": 2.4500000000000003e-06,
+      "loss": 2.235,
+      "step": 49
+    },
+    {
+      "epoch": 0.4573170731707317,
+      "grad_norm": 4.6343865394592285,
+      "learning_rate": 2.5e-06,
+      "loss": 2.1465,
+      "step": 50
+    },
+    {
+      "epoch": 0.46646341463414637,
+      "grad_norm": 4.669382095336914,
+      "learning_rate": 2.55e-06,
+      "loss": 2.0969,
+      "step": 51
+    },
+    {
+      "epoch": 0.47560975609756095,
+      "grad_norm": 4.207769870758057,
+      "learning_rate": 2.6e-06,
+      "loss": 2.0308,
+      "step": 52
+    },
+    {
+      "epoch": 0.4847560975609756,
+      "grad_norm": 5.222470760345459,
+      "learning_rate": 2.6500000000000005e-06,
+      "loss": 1.9445,
+      "step": 53
+    },
+    {
+      "epoch": 0.49390243902439024,
+      "grad_norm": 5.10925817489624,
+      "learning_rate": 2.7000000000000004e-06,
+      "loss": 1.7763,
+      "step": 54
+    },
+    {
+      "epoch": 0.5030487804878049,
+      "grad_norm": 4.8151044845581055,
+      "learning_rate": 2.7500000000000004e-06,
+      "loss": 1.7661,
+      "step": 55
+    },
+    {
+      "epoch": 0.5121951219512195,
+      "grad_norm": 6.155700206756592,
+      "learning_rate": 2.8000000000000003e-06,
+      "loss": 1.6665,
+      "step": 56
+    },
+    {
+      "epoch": 0.5213414634146342,
+      "grad_norm": 4.719021320343018,
+      "learning_rate": 2.85e-06,
+      "loss": 1.5532,
+      "step": 57
+    },
+    {
+      "epoch": 0.5304878048780488,
+      "grad_norm": 4.591799259185791,
+      "learning_rate": 2.9e-06,
+      "loss": 1.5244,
+      "step": 58
+    },
+    {
+      "epoch": 0.5396341463414634,
+      "grad_norm": 5.483795642852783,
+      "learning_rate": 2.95e-06,
+      "loss": 1.3816,
+      "step": 59
+    },
+    {
+      "epoch": 0.5487804878048781,
+      "grad_norm": 4.32553768157959,
+      "learning_rate": 3e-06,
+      "loss": 1.4268,
+      "step": 60
+    },
+    {
+      "epoch": 0.5579268292682927,
+      "grad_norm": 8.296231269836426,
+      "learning_rate": 3.05e-06,
+      "loss": 1.2887,
+      "step": 61
+    },
+    {
+      "epoch": 0.5670731707317073,
+      "grad_norm": 5.07594108581543,
+      "learning_rate": 3.1000000000000004e-06,
+      "loss": 1.228,
+      "step": 62
+    },
+    {
+      "epoch": 0.5762195121951219,
+      "grad_norm": 8.427435874938965,
+      "learning_rate": 3.1500000000000003e-06,
+      "loss": 1.1724,
+      "step": 63
+    },
+    {
+      "epoch": 0.5853658536585366,
+      "grad_norm": 22.45092010498047,
+      "learning_rate": 3.2000000000000003e-06,
+      "loss": 1.0871,
+      "step": 64
+    },
+    {
+      "epoch": 0.5945121951219512,
+      "grad_norm": 20.67412567138672,
+      "learning_rate": 3.2500000000000002e-06,
+      "loss": 1.0108,
+      "step": 65
+    },
+    {
+      "epoch": 0.6036585365853658,
+      "grad_norm": 4.921634674072266,
+      "learning_rate": 3.3000000000000006e-06,
+      "loss": 0.9775,
+      "step": 66
+    },
+    {
+      "epoch": 0.6128048780487805,
+      "grad_norm": 4.477286338806152,
+      "learning_rate": 3.3500000000000005e-06,
+      "loss": 0.9554,
+      "step": 67
+    },
+    {
+      "epoch": 0.6219512195121951,
+      "grad_norm": 8.190194129943848,
+      "learning_rate": 3.4000000000000005e-06,
+      "loss": 0.8583,
+      "step": 68
+    },
+    {
+      "epoch": 0.6310975609756098,
+      "grad_norm": 29.21201515197754,
+      "learning_rate": 3.45e-06,
+      "loss": 0.8499,
+      "step": 69
+    },
+    {
+      "epoch": 0.6402439024390244,
+      "grad_norm": 33.81047439575195,
+      "learning_rate": 3.5e-06,
+      "loss": 0.858,
+      "step": 70
+    },
+    {
+      "epoch": 0.649390243902439,
+      "grad_norm": 3.5125606060028076,
+      "learning_rate": 3.5500000000000003e-06,
+      "loss": 0.7227,
+      "step": 71
+    },
+    {
+      "epoch": 0.6585365853658537,
+      "grad_norm": 18.493423461914062,
+      "learning_rate": 3.6000000000000003e-06,
+      "loss": 0.7447,
+      "step": 72
+    },
+    {
+      "epoch": 0.6676829268292683,
+      "grad_norm": 24.923927307128906,
+      "learning_rate": 3.65e-06,
+      "loss": 0.8423,
+      "step": 73
+    },
+    {
+      "epoch": 0.676829268292683,
+      "grad_norm": 9.870631217956543,
+      "learning_rate": 3.7e-06,
+      "loss": 0.6934,
+      "step": 74
+    },
+    {
+      "epoch": 0.6859756097560976,
+      "grad_norm": 20.598569869995117,
+      "learning_rate": 3.7500000000000005e-06,
+      "loss": 0.7005,
+      "step": 75
+    },
+    {
+      "epoch": 0.6951219512195121,
+      "grad_norm": 30.662466049194336,
+      "learning_rate": 3.8000000000000005e-06,
+      "loss": 0.6226,
+      "step": 76
+    },
+    {
+      "epoch": 0.7042682926829268,
+      "grad_norm": 28.930774688720703,
+      "learning_rate": 3.85e-06,
+      "loss": 0.6357,
+      "step": 77
+    },
+    {
+      "epoch": 0.7134146341463414,
+      "grad_norm": 7.140467166900635,
+      "learning_rate": 3.900000000000001e-06,
+      "loss": 0.6129,
+      "step": 78
+    },
+    {
+      "epoch": 0.7225609756097561,
+      "grad_norm": 6.322911739349365,
+      "learning_rate": 3.95e-06,
+      "loss": 0.6086,
+      "step": 79
+    },
+    {
+      "epoch": 0.7317073170731707,
+      "grad_norm": 5.404141426086426,
+      "learning_rate": 4.000000000000001e-06,
+      "loss": 0.5986,
+      "step": 80
+    },
+    {
+      "epoch": 0.7408536585365854,
+      "grad_norm": 24.445228576660156,
+      "learning_rate": 4.05e-06,
+      "loss": 0.5932,
+      "step": 81
+    },
+    {
+      "epoch": 0.75,
+      "grad_norm": 14.275381088256836,
+      "learning_rate": 4.1e-06,
+      "loss": 0.5286,
+      "step": 82
+    },
+    {
+      "epoch": 0.7591463414634146,
+      "grad_norm": 9.77132511138916,
+      "learning_rate": 4.15e-06,
+      "loss": 0.4814,
+      "step": 83
+    },
+    {
+      "epoch": 0.7682926829268293,
+      "grad_norm": 11.043930053710938,
+      "learning_rate": 4.2000000000000004e-06,
+      "loss": 0.4953,
+      "step": 84
+    },
+    {
+      "epoch": 0.7774390243902439,
+      "grad_norm": 3.1333775520324707,
+      "learning_rate": 4.25e-06,
+      "loss": 0.4794,
+      "step": 85
+    },
+    {
+      "epoch": 0.7865853658536586,
+      "grad_norm": 9.50448989868164,
+      "learning_rate": 4.3e-06,
+      "loss": 0.4451,
+      "step": 86
+    },
+    {
+      "epoch": 0.7957317073170732,
+      "grad_norm": 5.867074489593506,
+      "learning_rate": 4.350000000000001e-06,
+      "loss": 0.4224,
+      "step": 87
+    },
+    {
+      "epoch": 0.8048780487804879,
+      "grad_norm": 12.556106567382812,
+      "learning_rate": 4.4e-06,
+      "loss": 0.4224,
+      "step": 88
+    },
+    {
+      "epoch": 0.8140243902439024,
+      "grad_norm": 7.916622638702393,
+      "learning_rate": 4.450000000000001e-06,
+      "loss": 0.4091,
+      "step": 89
+    },
+    {
+      "epoch": 0.823170731707317,
+      "grad_norm": 2.970932722091675,
+      "learning_rate": 4.5e-06,
+      "loss": 0.4181,
+      "step": 90
+    },
+    {
+      "epoch": 0.8323170731707317,
+      "grad_norm": 17.468870162963867,
+      "learning_rate": 4.5500000000000005e-06,
+      "loss": 0.4031,
+      "step": 91
+    },
+    {
+      "epoch": 0.8414634146341463,
+      "grad_norm": 15.861083984375,
+      "learning_rate": 4.600000000000001e-06,
+      "loss": 0.3604,
+      "step": 92
+    },
+    {
+      "epoch": 0.850609756097561,
+      "grad_norm": 6.715958118438721,
+      "learning_rate": 4.65e-06,
+      "loss": 0.3489,
+      "step": 93
+    },
+    {
+      "epoch": 0.8597560975609756,
+      "grad_norm": 5.439844608306885,
+      "learning_rate": 4.7e-06,
+      "loss": 0.3339,
+      "step": 94
+    },
+    {
+      "epoch": 0.8689024390243902,
+      "grad_norm": 4.650434494018555,
+      "learning_rate": 4.75e-06,
+      "loss": 0.3189,
+      "step": 95
+    },
+    {
+      "epoch": 0.8780487804878049,
+      "grad_norm": 15.319528579711914,
+      "learning_rate": 4.800000000000001e-06,
+      "loss": 0.3347,
+      "step": 96
+    },
+    {
+      "epoch": 0.8871951219512195,
+      "grad_norm": 5.151925563812256,
+      "learning_rate": 4.85e-06,
+      "loss": 0.3043,
+      "step": 97
+    },
+    {
+      "epoch": 0.8963414634146342,
+      "grad_norm": 6.351178169250488,
+      "learning_rate": 4.9000000000000005e-06,
+      "loss": 0.2939,
+      "step": 98
+    },
+    {
+      "epoch": 0.9054878048780488,
+      "grad_norm": 3.942911148071289,
+      "learning_rate": 4.95e-06,
+      "loss": 0.2887,
+      "step": 99
+    },
+    {
+      "epoch": 0.9146341463414634,
+      "grad_norm": 3.0885493755340576,
+      "learning_rate": 5e-06,
+      "loss": 0.2992,
+      "step": 100
+    },
+    {
+      "epoch": 0.9237804878048781,
+      "grad_norm": 7.023274898529053,
+      "learning_rate": 4.999959803423605e-06,
+      "loss": 0.2607,
+      "step": 101
+    },
+    {
+      "epoch": 0.9329268292682927,
+      "grad_norm": 8.774216651916504,
+      "learning_rate": 4.9998392149870315e-06,
+      "loss": 0.2674,
+      "step": 102
+    },
+    {
+      "epoch": 0.9420731707317073,
+      "grad_norm": 3.5439791679382324,
+      "learning_rate": 4.999638238568072e-06,
+      "loss": 0.2602,
+      "step": 103
+    },
+    {
+      "epoch": 0.9512195121951219,
+      "grad_norm": 7.669381618499756,
+      "learning_rate": 4.999356880629579e-06,
+      "loss": 0.2718,
+      "step": 104
+    },
+    {
+      "epoch": 0.9603658536585366,
+      "grad_norm": 6.9407854080200195,
+      "learning_rate": 4.998995150219252e-06,
+      "loss": 0.2494,
+      "step": 105
+    },
+    {
+      "epoch": 0.9695121951219512,
+      "grad_norm": 12.144411087036133,
+      "learning_rate": 4.998553058969352e-06,
+      "loss": 0.2297,
+      "step": 106
+    },
+    {
+      "epoch": 0.9786585365853658,
+      "grad_norm": 6.956138610839844,
+      "learning_rate": 4.998030621096322e-06,
+      "loss": 0.2386,
+      "step": 107
+    },
+    {
+      "epoch": 0.9878048780487805,
+      "grad_norm": 8.304037094116211,
+      "learning_rate": 4.997427853400333e-06,
+      "loss": 0.23,
+      "step": 108
+    },
+    {
+      "epoch": 0.9969512195121951,
+      "grad_norm": 7.031365394592285,
+      "learning_rate": 4.996744775264743e-06,
+      "loss": 0.2346,
+      "step": 109
+    },
+    {
+      "epoch": 1.0,
+      "grad_norm": 7.031365394592285,
+      "learning_rate": 4.9959814086554735e-06,
+      "loss": 0.0739,
+      "step": 110
+    },
+    {
+      "epoch": 1.0091463414634145,
+      "grad_norm": 2.084608793258667,
+      "learning_rate": 4.995137778120306e-06,
+      "loss": 0.2422,
+      "step": 111
+    },
+    {
+      "epoch": 1.0182926829268293,
+      "grad_norm": 38.12245178222656,
+      "learning_rate": 4.994213910788085e-06,
+      "loss": 0.1992,
+      "step": 112
+    },
+    {
+      "epoch": 1.0274390243902438,
+      "grad_norm": 2.817565441131592,
+      "learning_rate": 4.993209836367856e-06,
+      "loss": 0.1882,
+      "step": 113
+    },
+    {
+      "epoch": 1.0365853658536586,
+      "grad_norm": 11.774249076843262,
+      "learning_rate": 4.9921255871479e-06,
+      "loss": 0.2235,
+      "step": 114
+    },
+    {
+      "epoch": 1.045731707317073,
+      "grad_norm": 2.3362019062042236,
+      "learning_rate": 4.9909611979947046e-06,
+      "loss": 0.1875,
+      "step": 115
+    },
+    {
+      "epoch": 1.0548780487804879,
+      "grad_norm": 1.9516818523406982,
+      "learning_rate": 4.989716706351835e-06,
+      "loss": 0.1858,
+      "step": 116
+    },
+    {
+      "epoch": 1.0640243902439024,
+      "grad_norm": 1.5484645366668701,
+      "learning_rate": 4.9883921522387336e-06,
+      "loss": 0.1845,
+      "step": 117
+    },
+    {
+      "epoch": 1.0731707317073171,
+      "grad_norm": 7.430396556854248,
+      "learning_rate": 4.986987578249433e-06,
+      "loss": 0.184,
+      "step": 118
+    },
+    {
+      "epoch": 1.0823170731707317,
+      "grad_norm": 2.0725948810577393,
+      "learning_rate": 4.9855030295511845e-06,
+      "loss": 0.177,
+      "step": 119
+    },
+    {
+      "epoch": 1.0914634146341464,
+      "grad_norm": 4.836520671844482,
+      "learning_rate": 4.98393855388301e-06,
+      "loss": 0.1744,
+      "step": 120
+    },
+    {
+      "epoch": 1.100609756097561,
+      "grad_norm": 5.22581672668457,
+      "learning_rate": 4.982294201554162e-06,
+      "loss": 0.1599,
+      "step": 121
+    },
+    {
+      "epoch": 1.1097560975609757,
+      "grad_norm": 5.406208038330078,
+      "learning_rate": 4.980570025442507e-06,
+      "loss": 0.1645,
+      "step": 122
+    },
+    {
+      "epoch": 1.1189024390243902,
+      "grad_norm": 8.736640930175781,
+      "learning_rate": 4.978766080992827e-06,
+      "loss": 0.1943,
+      "step": 123
+    },
+    {
+      "epoch": 1.1280487804878048,
+      "grad_norm": 9.184198379516602,
+      "learning_rate": 4.976882426215035e-06,
+      "loss": 0.1753,
+      "step": 124
+    },
+    {
+      "epoch": 1.1371951219512195,
+      "grad_norm": 5.317800521850586,
+      "learning_rate": 4.974919121682307e-06,
+      "loss": 0.1897,
+      "step": 125
+    },
+    {
+      "epoch": 1.146341463414634,
+      "grad_norm": 11.803364753723145,
+      "learning_rate": 4.972876230529143e-06,
+      "loss": 0.1516,
+      "step": 126
+    },
+    {
+      "epoch": 1.1554878048780488,
+      "grad_norm": 31.74310302734375,
+      "learning_rate": 4.9707538184493255e-06,
+      "loss": 0.1947,
+      "step": 127
+    },
+    {
+      "epoch": 1.1646341463414633,
+      "grad_norm": 1.956932783126831,
+      "learning_rate": 4.968551953693813e-06,
+      "loss": 0.1708,
+      "step": 128
+    },
+    {
+      "epoch": 1.173780487804878,
+      "grad_norm": 4.64445686340332,
+      "learning_rate": 4.966270707068548e-06,
+      "loss": 0.1711,
+      "step": 129
+    },
+    {
+      "epoch": 1.1829268292682926,
+      "grad_norm": 4.442819595336914,
+      "learning_rate": 4.963910151932171e-06,
+      "loss": 0.1572,
+      "step": 130
+    },
+    {
+      "epoch": 1.1920731707317074,
+      "grad_norm": 1.9702759981155396,
+      "learning_rate": 4.961470364193673e-06,
+      "loss": 0.1364,
+      "step": 131
+    },
+    {
+      "epoch": 1.201219512195122,
+      "grad_norm": 7.816587448120117,
+      "learning_rate": 4.9589514223099425e-06,
+      "loss": 0.1624,
+      "step": 132
+    },
+    {
+      "epoch": 1.2103658536585367,
+      "grad_norm": 61.18424606323242,
+      "learning_rate": 4.956353407283252e-06,
+      "loss": 0.26,
+      "step": 133
+    },
+    {
+      "epoch": 1.2195121951219512,
+      "grad_norm": 15.649847984313965,
+      "learning_rate": 4.9536764026586505e-06,
+      "loss": 0.1543,
+      "step": 134
+    },
+    {
+      "epoch": 1.228658536585366,
+      "grad_norm": 2.249363660812378,
+      "learning_rate": 4.950920494521274e-06,
+      "loss": 0.152,
+      "step": 135
+    },
+    {
+      "epoch": 1.2378048780487805,
+      "grad_norm": 4.055661678314209,
+      "learning_rate": 4.948085771493579e-06,
+      "loss": 0.1402,
+      "step": 136
+    },
+    {
+      "epoch": 1.2469512195121952,
+      "grad_norm": 2.2803292274475098,
+      "learning_rate": 4.945172324732496e-06,
+      "loss": 0.1427,
+      "step": 137
+    },
+    {
+      "epoch": 1.2560975609756098,
+      "grad_norm": 2.5540456771850586,
+      "learning_rate": 4.942180247926492e-06,
+      "loss": 0.143,
+      "step": 138
+    },
+    {
+      "epoch": 1.2652439024390243,
+      "grad_norm": 8.80340576171875,
+      "learning_rate": 4.939109637292563e-06,
+      "loss": 0.1569,
+      "step": 139
+    },
+    {
+      "epoch": 1.274390243902439,
+      "grad_norm": 3.1077523231506348,
+      "learning_rate": 4.935960591573136e-06,
+      "loss": 0.1313,
+      "step": 140
+    },
+    {
+      "epoch": 1.2835365853658536,
+      "grad_norm": 1.2506815195083618,
+      "learning_rate": 4.932733212032896e-06,
+      "loss": 0.137,
+      "step": 141
+    },
+    {
+      "epoch": 1.2926829268292683,
+      "grad_norm": 2.4397666454315186,
+      "learning_rate": 4.929427602455532e-06,
+      "loss": 0.1408,
+      "step": 142
+    },
+    {
+      "epoch": 1.3018292682926829,
+      "grad_norm": 1.7658663988113403,
+      "learning_rate": 4.926043869140392e-06,
+      "loss": 0.133,
+      "step": 143
+    },
+    {
+      "epoch": 1.3109756097560976,
+      "grad_norm": 1.0473108291625977,
+      "learning_rate": 4.922582120899072e-06,
+      "loss": 0.1358,
+      "step": 144
+    },
+    {
+      "epoch": 1.3201219512195121,
+      "grad_norm": 5.566607475280762,
+      "learning_rate": 4.919042469051916e-06,
+      "loss": 0.1237,
+      "step": 145
+    },
+    {
+      "epoch": 1.329268292682927,
+      "grad_norm": 1.0360139608383179,
+      "learning_rate": 4.915425027424429e-06,
+      "loss": 0.1274,
+      "step": 146
+    },
+    {
+      "epoch": 1.3384146341463414,
+      "grad_norm": 1.9091176986694336,
+      "learning_rate": 4.911729912343631e-06,
+      "loss": 0.1439,
+      "step": 147
+    },
+    {
+      "epoch": 1.3475609756097562,
+      "grad_norm": 1.5727638006210327,
+      "learning_rate": 4.907957242634299e-06,
+      "loss": 0.1259,
+      "step": 148
+    },
+    {
+      "epoch": 1.3567073170731707,
+      "grad_norm": 1.0574792623519897,
+      "learning_rate": 4.9041071396151585e-06,
+      "loss": 0.142,
+      "step": 149
+    },
+    {
+      "epoch": 1.3658536585365852,
+      "grad_norm": 8.020130157470703,
+      "learning_rate": 4.900179727094978e-06,
+      "loss": 0.1392,
+      "step": 150
+    },
+    {
+      "epoch": 1.375,
+      "grad_norm": 3.156255006790161,
+      "learning_rate": 4.896175131368587e-06,
+      "loss": 0.152,
+      "step": 151
+    },
+    {
+      "epoch": 1.3841463414634148,
+      "grad_norm": 2.3956990242004395,
+      "learning_rate": 4.892093481212817e-06,
+      "loss": 0.1244,
+      "step": 152
+    },
+    {
+      "epoch": 1.3932926829268293,
+      "grad_norm": 2.6058578491210938,
+      "learning_rate": 4.887934907882357e-06,
+      "loss": 0.1271,
+      "step": 153
+    },
+    {
+      "epoch": 1.4024390243902438,
+      "grad_norm": 1.7173289060592651,
+      "learning_rate": 4.883699545105535e-06,
+      "loss": 0.1257,
+      "step": 154
+    },
+    {
+      "epoch": 1.4115853658536586,
+      "grad_norm": 2.2237367630004883,
+      "learning_rate": 4.8793875290800186e-06,
+      "loss": 0.127,
+      "step": 155
+    },
+    {
+      "epoch": 1.420731707317073,
+      "grad_norm": 6.997623920440674,
+      "learning_rate": 4.874998998468433e-06,
+      "loss": 0.1454,
+      "step": 156
+    },
+    {
+      "epoch": 1.4298780487804879,
+      "grad_norm": 10.645139694213867,
+      "learning_rate": 4.870534094393903e-06,
+      "loss": 0.1402,
+      "step": 157
+    },
+    {
+      "epoch": 1.4390243902439024,
+      "grad_norm": 1.5808072090148926,
+      "learning_rate": 4.865992960435514e-06,
+      "loss": 0.1156,
+      "step": 158
+    },
+    {
+      "epoch": 1.4481707317073171,
+      "grad_norm": 0.990803062915802,
+      "learning_rate": 4.861375742623697e-06,
+      "loss": 0.1286,
+      "step": 159
+    },
+    {
+      "epoch": 1.4573170731707317,
+      "grad_norm": 1.5008662939071655,
+      "learning_rate": 4.856682589435531e-06,
+      "loss": 0.1282,
+      "step": 160
+    },
+    {
+      "epoch": 1.4664634146341464,
+      "grad_norm": 1.772304892539978,
+      "learning_rate": 4.851913651789968e-06,
+      "loss": 0.1308,
+      "step": 161
+    },
+    {
+      "epoch": 1.475609756097561,
+      "grad_norm": 1.201061487197876,
+      "learning_rate": 4.847069083042982e-06,
+      "loss": 0.1198,
+      "step": 162
+    },
+    {
+      "epoch": 1.4847560975609757,
+      "grad_norm": 0.6147828698158264,
+      "learning_rate": 4.8421490389826344e-06,
+      "loss": 0.1138,
+      "step": 163
+    },
+    {
+      "epoch": 1.4939024390243902,
+      "grad_norm": 1.3990256786346436,
+      "learning_rate": 4.837153677824068e-06,
+      "loss": 0.1205,
+      "step": 164
+    },
+    {
+      "epoch": 1.5030487804878048,
+      "grad_norm": 1.6977574825286865,
+      "learning_rate": 4.832083160204414e-06,
+      "loss": 0.1128,
+      "step": 165
+    },
+    {
+      "epoch": 1.5121951219512195,
+      "grad_norm": 2.4056453704833984,
+      "learning_rate": 4.826937649177632e-06,
+      "loss": 0.1184,
+      "step": 166
+    },
+    {
+      "epoch": 1.5213414634146343,
+      "grad_norm": 2.7455694675445557,
+      "learning_rate": 4.821717310209265e-06,
+      "loss": 0.123,
+      "step": 167
+    },
+    {
+      "epoch": 1.5304878048780488,
+      "grad_norm": 1.8243584632873535,
+      "learning_rate": 4.816422311171115e-06,
+      "loss": 0.1198,
+      "step": 168
+    },
+    {
+      "epoch": 1.5396341463414633,
+      "grad_norm": 0.8920577764511108,
+      "learning_rate": 4.811052822335849e-06,
+      "loss": 0.1088,
+      "step": 169
+    },
+    {
+      "epoch": 1.548780487804878,
+      "grad_norm": 1.6639117002487183,
+      "learning_rate": 4.805609016371522e-06,
+      "loss": 0.1231,
+      "step": 170
+    },
+    {
+      "epoch": 1.5579268292682928,
+      "grad_norm": 1.383026123046875,
+      "learning_rate": 4.800091068336024e-06,
+      "loss": 0.1207,
+      "step": 171
+    },
+    {
+      "epoch": 1.5670731707317072,
+      "grad_norm": 1.6599416732788086,
+      "learning_rate": 4.79449915567145e-06,
+      "loss": 0.1206,
+      "step": 172
+    },
+    {
+      "epoch": 1.576219512195122,
+      "grad_norm": 1.5868676900863647,
+      "learning_rate": 4.788833458198395e-06,
+      "loss": 0.1158,
+      "step": 173
+    },
+    {
+      "epoch": 1.5853658536585367,
+      "grad_norm": 1.0154211521148682,
+      "learning_rate": 4.783094158110175e-06,
+      "loss": 0.1091,
+      "step": 174
+    },
+    {
+      "epoch": 1.5945121951219512,
+      "grad_norm": 2.1514394283294678,
+      "learning_rate": 4.7772814399669585e-06,
+      "loss": 0.1109,
+      "step": 175
+    },
+    {
+      "epoch": 1.6036585365853657,
+      "grad_norm": 0.7475085258483887,
+      "learning_rate": 4.7713954906898415e-06,
+      "loss": 0.1162,
+      "step": 176
+    },
+    {
+      "epoch": 1.6128048780487805,
+      "grad_norm": 0.8523163795471191,
+      "learning_rate": 4.7654364995548325e-06,
+      "loss": 0.1238,
+      "step": 177
+    },
+    {
+      "epoch": 1.6219512195121952,
+      "grad_norm": 1.5456233024597168,
+      "learning_rate": 4.759404658186765e-06,
+      "loss": 0.1163,
+      "step": 178
+    },
+    {
+      "epoch": 1.6310975609756098,
+      "grad_norm": 1.123340129852295,
+      "learning_rate": 4.753300160553137e-06,
+      "loss": 0.1104,
+      "step": 179
+    },
+    {
+      "epoch": 1.6402439024390243,
+      "grad_norm": 1.16841721534729,
+      "learning_rate": 4.747123202957871e-06,
+      "loss": 0.1099,
+      "step": 180
+    },
+    {
+      "epoch": 1.649390243902439,
+      "grad_norm": 2.025221109390259,
+      "learning_rate": 4.740873984035009e-06,
+      "loss": 0.1157,
+      "step": 181
+    },
+    {
+      "epoch": 1.6585365853658538,
+      "grad_norm": 3.876469373703003,
+      "learning_rate": 4.734552704742314e-06,
+      "loss": 0.1122,
+      "step": 182
+    },
+    {
+      "epoch": 1.6676829268292683,
+      "grad_norm": 1.5562145709991455,
+      "learning_rate": 4.728159568354814e-06,
+      "loss": 0.1127,
+      "step": 183
+    },
+    {
+      "epoch": 1.6768292682926829,
+      "grad_norm": 1.4691755771636963,
+      "learning_rate": 4.721694780458266e-06,
+      "loss": 0.1182,
+      "step": 184
+    },
+    {
+      "epoch": 1.6859756097560976,
+      "grad_norm": 1.9529973268508911,
+      "learning_rate": 4.715158548942541e-06,
+      "loss": 0.1153,
+      "step": 185
+    },
+    {
+      "epoch": 1.6951219512195121,
+      "grad_norm": 1.9065771102905273,
+      "learning_rate": 4.708551083994944e-06,
+      "loss": 0.1163,
+      "step": 186
+    },
+    {
+      "epoch": 1.7042682926829267,
+      "grad_norm": 1.0964735746383667,
+      "learning_rate": 4.701872598093452e-06,
+      "loss": 0.114,
+      "step": 187
+    },
+    {
+      "epoch": 1.7134146341463414,
+      "grad_norm": 1.0885926485061646,
+      "learning_rate": 4.695123305999877e-06,
+      "loss": 0.114,
+      "step": 188
+    },
+    {
+      "epoch": 1.7225609756097562,
+      "grad_norm": 1.1777565479278564,
+      "learning_rate": 4.688303424752969e-06,
+      "loss": 0.1116,
+      "step": 189
+    },
+    {
+      "epoch": 1.7317073170731707,
+      "grad_norm": 1.3686976432800293,
+      "learning_rate": 4.6814131736614286e-06,
+      "loss": 0.1081,
+      "step": 190
+    },
+    {
+      "epoch": 1.7408536585365852,
+      "grad_norm": 0.8220391869544983,
+      "learning_rate": 4.674452774296861e-06,
+      "loss": 0.1091,
+      "step": 191
+    },
+    {
+      "epoch": 1.75,
+      "grad_norm": 1.073935866355896,
+      "learning_rate": 4.667422450486646e-06,
+      "loss": 0.1148,
+      "step": 192
+    },
+    {
+      "epoch": 1.7591463414634148,
+      "grad_norm": 1.389614224433899,
+      "learning_rate": 4.660322428306741e-06,
+      "loss": 0.1134,
+      "step": 193
+    },
+    {
+      "epoch": 1.7682926829268293,
+      "grad_norm": 1.349023699760437,
+      "learning_rate": 4.653152936074413e-06,
+      "loss": 0.1056,
+      "step": 194
+    },
+    {
+      "epoch": 1.7774390243902438,
+      "grad_norm": 1.0334906578063965,
+      "learning_rate": 4.645914204340898e-06,
+      "loss": 0.113,
+      "step": 195
+    },
+    {
+      "epoch": 1.7865853658536586,
+      "grad_norm": 18.6312313079834,
+      "learning_rate": 4.638606465883979e-06,
+      "loss": 0.1351,
+      "step": 196
+    },
+    {
+      "epoch": 1.7957317073170733,
+      "grad_norm": 0.7669345736503601,
+      "learning_rate": 4.6312299557005125e-06,
+      "loss": 0.1097,
+      "step": 197
+    },
+    {
+      "epoch": 1.8048780487804879,
+      "grad_norm": 2.1091127395629883,
+      "learning_rate": 4.623784910998862e-06,
+      "loss": 0.1122,
+      "step": 198
+    },
+    {
+      "epoch": 1.8140243902439024,
+      "grad_norm": 0.8031914830207825,
+      "learning_rate": 4.616271571191273e-06,
+      "loss": 0.1109,
+      "step": 199
+    },
+    {
+      "epoch": 1.8231707317073171,
+      "grad_norm": 2.1154661178588867,
+      "learning_rate": 4.608690177886177e-06,
+      "loss": 0.1131,
+      "step": 200
+    },
+    {
+      "epoch": 1.8323170731707317,
+      "grad_norm": 1.4946709871292114,
+      "learning_rate": 4.6010409748804165e-06,
+      "loss": 0.1136,
+      "step": 201
+    },
+    {
+      "epoch": 1.8414634146341462,
+      "grad_norm": 1.0509642362594604,
+      "learning_rate": 4.593324208151412e-06,
+      "loss": 0.1162,
+      "step": 202
+    },
+    {
+      "epoch": 1.850609756097561,
+      "grad_norm": 0.8742370009422302,
+      "learning_rate": 4.585540125849244e-06,
+      "loss": 0.1102,
+      "step": 203
+    },
+    {
+      "epoch": 1.8597560975609757,
+      "grad_norm": 1.1283056735992432,
+      "learning_rate": 4.577688978288681e-06,
+      "loss": 0.1104,
+      "step": 204
+    },
+    {
+      "epoch": 1.8689024390243902,
+      "grad_norm": 1.0081048011779785,
+      "learning_rate": 4.569771017941125e-06,
+      "loss": 0.1119,
+      "step": 205
+    },
+    {
+      "epoch": 1.8780487804878048,
+      "grad_norm": 0.7291656732559204,
+      "learning_rate": 4.561786499426494e-06,
+      "loss": 0.1103,
+      "step": 206
+    },
+    {
+      "epoch": 1.8871951219512195,
+      "grad_norm": 1.144096851348877,
+      "learning_rate": 4.5537356795050346e-06,
+      "loss": 0.1125,
+      "step": 207
+    },
+    {
+      "epoch": 1.8963414634146343,
+      "grad_norm": 0.7317456603050232,
+      "learning_rate": 4.545618817069066e-06,
+      "loss": 0.1105,
+      "step": 208
+    },
+    {
+      "epoch": 1.9054878048780488,
+      "grad_norm": 1.1636486053466797,
+      "learning_rate": 4.537436173134653e-06,
+      "loss": 0.1091,
+      "step": 209
+    },
+    {
+      "epoch": 1.9146341463414633,
+      "grad_norm": 0.6319037675857544,
+      "learning_rate": 4.529188010833212e-06,
+      "loss": 0.1088,
+      "step": 210
+    },
+    {
+      "epoch": 1.923780487804878,
+      "grad_norm": 0.7535589933395386,
+      "learning_rate": 4.520874595403053e-06,
+      "loss": 0.1041,
+      "step": 211
+    },
+    {
+      "epoch": 1.9329268292682928,
+      "grad_norm": 1.0461041927337646,
+      "learning_rate": 4.512496194180846e-06,
+      "loss": 0.1129,
+      "step": 212
+    },
+    {
+      "epoch": 1.9420731707317072,
+      "grad_norm": 0.6124874949455261,
+      "learning_rate": 4.5040530765930275e-06,
+      "loss": 0.1113,
+      "step": 213
+    },
+    {
+      "epoch": 1.951219512195122,
+      "grad_norm": 0.8516119122505188,
+      "learning_rate": 4.495545514147134e-06,
+      "loss": 0.1112,
+      "step": 214
+    },
+    {
+      "epoch": 1.9603658536585367,
+      "grad_norm": 0.8919075131416321,
+      "learning_rate": 4.486973780423073e-06,
+      "loss": 0.1089,
+      "step": 215
+    },
+    {
+      "epoch": 1.9695121951219512,
+      "grad_norm": 1.2183221578598022,
+      "learning_rate": 4.478338151064323e-06,
+      "loss": 0.1156,
+      "step": 216
+    },
+    {
+      "epoch": 1.9786585365853657,
+      "grad_norm": 1.0123646259307861,
+      "learning_rate": 4.4696389037690734e-06,
+      "loss": 0.1077,
+      "step": 217
+    },
+    {
+      "epoch": 1.9878048780487805,
+      "grad_norm": 0.8304896950721741,
+      "learning_rate": 4.460876318281291e-06,
+      "loss": 0.1051,
+      "step": 218
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 654,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 6,
+  "save_steps": 109,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 6.596238386615288e+18,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}
diff --git a/checkpoint-218/training_args.bin b/checkpoint-218/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..24b9be216f1966bb6d081a79c30a21dced39cf99
--- /dev/null
+++ b/checkpoint-218/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:14990b8399bb8d39b746405b094f7eea1041d93488de8433b2d415de16ff957a
+size 7928
diff --git a/checkpoint-218/zero_to_fp32.py b/checkpoint-218/zero_to_fp32.py
new file mode 100644
index 0000000000000000000000000000000000000000..24cc342e78d1a006c782b3a4cd68d9ce786d8fd8
--- /dev/null
+++ b/checkpoint-218/zero_to_fp32.py
@@ -0,0 +1,604 @@
+#!/usr/bin/env python
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
+# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
+# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
+# application.
+#
+# example: python zero_to_fp32.py . pytorch_model.bin
+
+import argparse
+import torch
+import glob
+import math
+import os
+import re
+from collections import OrderedDict
+from dataclasses import dataclass
+
+# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
+# DeepSpeed data structures it has to be available in the current python environment.
+from deepspeed.utils import logger
+from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
+                                            FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
+                                            FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS)
+
+
+@dataclass
+class zero_model_state:
+    buffers: dict()
+    param_shapes: dict()
+    shared_params: list
+    ds_version: int
+    frozen_param_shapes: dict()
+    frozen_param_fragments: dict()
+
+
+debug = 0
+
+# load to cpu
+device = torch.device('cpu')
+
+
+def atoi(text):
+    return int(text) if text.isdigit() else text
+
+
+def natural_keys(text):
+    '''
+    alist.sort(key=natural_keys) sorts in human order
+    http://nedbatchelder.com/blog/200712/human_sorting.html
+    (See Toothy's implementation in the comments)
+    '''
+    return [atoi(c) for c in re.split(r'(\d+)', text)]
+
+
+def get_model_state_file(checkpoint_dir, zero_stage):
+    if not os.path.isdir(checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
+
+    # there should be only one file
+    if zero_stage <= 2:
+        file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
+    elif zero_stage == 3:
+        file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
+
+    if not os.path.exists(file):
+        raise FileNotFoundError(f"can't find model states file at '{file}'")
+
+    return file
+
+
+def get_checkpoint_files(checkpoint_dir, glob_pattern):
+    # XXX: need to test that this simple glob rule works for multi-node setup too
+    ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
+
+    if len(ckpt_files) == 0:
+        raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
+
+    return ckpt_files
+
+
+def get_optim_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
+
+
+def get_model_state_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
+
+
+def parse_model_states(files):
+    zero_model_states = []
+    for file in files:
+        state_dict = torch.load(file, map_location=device)
+
+        if BUFFER_NAMES not in state_dict:
+            raise ValueError(f"{file} is not a model state checkpoint")
+        buffer_names = state_dict[BUFFER_NAMES]
+        if debug:
+            print("Found buffers:", buffer_names)
+
+        # recover just the buffers while restoring them to fp32 if they were saved in fp16
+        buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
+        param_shapes = state_dict[PARAM_SHAPES]
+
+        # collect parameters that are included in param_shapes
+        param_names = []
+        for s in param_shapes:
+            for name in s.keys():
+                param_names.append(name)
+
+        # update with frozen parameters
+        frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
+        if frozen_param_shapes is not None:
+            if debug:
+                print(f"Found frozen_param_shapes: {frozen_param_shapes}")
+            param_names += list(frozen_param_shapes.keys())
+
+        # handle shared params
+        shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
+
+        ds_version = state_dict.get(DS_VERSION, None)
+
+        frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
+
+        z_model_state = zero_model_state(buffers=buffers,
+                                         param_shapes=param_shapes,
+                                         shared_params=shared_params,
+                                         ds_version=ds_version,
+                                         frozen_param_shapes=frozen_param_shapes,
+                                         frozen_param_fragments=frozen_param_fragments)
+        zero_model_states.append(z_model_state)
+
+    return zero_model_states
+
+
+def parse_optim_states(files, ds_checkpoint_dir):
+
+    total_files = len(files)
+    state_dicts = []
+    for f in files:
+        state_dict = torch.load(f, map_location=device)
+        # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+        # and also handle the case where it was already removed by another helper script
+        state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+        state_dicts.append(state_dict)
+
+    if not ZERO_STAGE in state_dicts[0][OPTIMIZER_STATE_DICT]:
+        raise ValueError(f"{files[0]} is not a zero checkpoint")
+    zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
+    world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
+
+    # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
+    # parameters can be different from data parallelism for non-expert parameters. So we can just
+    # use the max of the partition_count to get the dp world_size.
+
+    if type(world_size) is list:
+        world_size = max(world_size)
+
+    if world_size != total_files:
+        raise ValueError(
+            f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
+            "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
+        )
+
+    # the groups are named differently in each stage
+    if zero_stage <= 2:
+        fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
+    elif zero_stage == 3:
+        fp32_groups_key = FP32_FLAT_GROUPS
+    else:
+        raise ValueError(f"unknown zero stage {zero_stage}")
+
+    if zero_stage <= 2:
+        fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
+    elif zero_stage == 3:
+        # if there is more than one param group, there will be multiple flattened tensors - one
+        # flattened tensor per group - for simplicity merge them into a single tensor
+        #
+        # XXX: could make the script more memory efficient for when there are multiple groups - it
+        # will require matching the sub-lists of param_shapes for each param group flattened tensor
+
+        fp32_flat_groups = [
+            torch.cat(state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key], 0) for i in range(len(state_dicts))
+        ]
+
+    return zero_stage, world_size, fp32_flat_groups
+
+
+def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
+    """
+    Returns fp32 state_dict reconstructed from ds checkpoint
+
+    Args:
+        - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
+
+    """
+    print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
+
+    optim_files = get_optim_files(ds_checkpoint_dir)
+    zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
+    print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
+
+    model_files = get_model_state_files(ds_checkpoint_dir)
+
+    zero_model_states = parse_model_states(model_files)
+    print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
+
+    if zero_stage <= 2:
+        return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+    elif zero_stage == 3:
+        return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+
+
+def _zero2_merge_frozen_params(state_dict, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+    frozen_param_fragments = zero_model_states[0].frozen_param_fragments
+
+    if debug:
+        num_elem = sum(s.numel() for s in frozen_param_shapes.values())
+        print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        state_dict[name] = frozen_param_fragments[name]
+
+        if debug:
+            print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _has_callable(obj, fn):
+    attr = getattr(obj, fn, None)
+    return callable(attr)
+
+
+def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+
+    # Reconstruction protocol:
+    #
+    # XXX: document this
+
+    if debug:
+        for i in range(world_size):
+            for j in range(len(fp32_flat_groups[0])):
+                print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
+
+    # XXX: memory usage doubles here (zero2)
+    num_param_groups = len(fp32_flat_groups[0])
+    merged_single_partition_of_fp32_groups = []
+    for i in range(num_param_groups):
+        merged_partitions = [sd[i] for sd in fp32_flat_groups]
+        full_single_fp32_vector = torch.cat(merged_partitions, 0)
+        merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
+    avail_numel = sum(
+        [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
+
+    if debug:
+        wanted_params = sum([len(shapes) for shapes in param_shapes])
+        wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
+        # not asserting if there is a mismatch due to possible padding
+        print(f"Have {avail_numel} numels to process.")
+        print(f"Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    total_numel = 0
+    total_params = 0
+    for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
+        offset = 0
+        avail_numel = full_single_fp32_vector.numel()
+        for name, shape in shapes.items():
+
+            unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
+            total_numel += unpartitioned_numel
+            total_params += 1
+
+            if debug:
+                print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+            state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
+            offset += unpartitioned_numel
+
+        # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
+        # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
+        # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
+        # live optimizer object, so we are checking that the numbers are within the right range
+        align_to = 2 * world_size
+
+        def zero2_align(x):
+            return align_to * math.ceil(x / align_to)
+
+        if debug:
+            print(f"original offset={offset}, avail_numel={avail_numel}")
+
+        offset = zero2_align(offset)
+        avail_numel = zero2_align(avail_numel)
+
+        if debug:
+            print(f"aligned  offset={offset}, avail_numel={avail_numel}")
+
+        # Sanity check
+        if offset != avail_numel:
+            raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero2_merge_frozen_params(state_dict, zero_model_states)
+
+    _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def zero3_partitioned_param_info(unpartitioned_numel, world_size):
+    remainder = unpartitioned_numel % world_size
+    padding_numel = (world_size - remainder) if remainder else 0
+    partitioned_numel = math.ceil(unpartitioned_numel / world_size)
+    return partitioned_numel, padding_numel
+
+
+def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    if debug:
+        for i in range(world_size):
+            num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
+            print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in zero_model_states[0].frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
+        state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
+
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+    avail_numel = fp32_flat_groups[0].numel() * world_size
+    # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
+    # param, re-consolidating each param, while dealing with padding if any
+
+    # merge list of dicts, preserving order
+    param_shapes = {k: v for d in param_shapes for k, v in d.items()}
+
+    if debug:
+        for i in range(world_size):
+            print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
+
+        wanted_params = len(param_shapes)
+        wanted_numel = sum(shape.numel() for shape in param_shapes.values())
+        # not asserting if there is a mismatch due to possible padding
+        avail_numel = fp32_flat_groups[0].numel() * world_size
+        print(f"Trainable params: Have {avail_numel} numels to process.")
+        print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    offset = 0
+    total_numel = 0
+    total_params = 0
+    for name, shape in param_shapes.items():
+
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+        total_params += 1
+
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+        # XXX: memory usage doubles here
+        state_dict[name] = torch.cat(
+            tuple(fp32_flat_groups[i].narrow(0, offset, partitioned_numel) for i in range(world_size)),
+            0).narrow(0, 0, unpartitioned_numel).view(shape)
+        offset += partitioned_numel
+
+    offset *= world_size
+
+    # Sanity check
+    if offset != avail_numel:
+        raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
+
+    _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag=None, exclude_frozen_parameters=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
+    ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
+    via a model hub.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+
+    Returns:
+        - pytorch ``state_dict``
+
+    Note: this approach may not work if your application doesn't have sufficient free CPU memory and
+    you may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
+    the checkpoint.
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+        # do the training and checkpoint saving
+        state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
+        model = model.cpu() # move to cpu
+        model.load_state_dict(state_dict)
+        # submit to model hub or save the model to share with others
+
+    In this example the ``model`` will no longer be usable in the deepspeed context of the same
+    application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
+
+    """
+    if tag is None:
+        latest_path = os.path.join(checkpoint_dir, 'latest')
+        if os.path.isfile(latest_path):
+            with open(latest_path, 'r') as fd:
+                tag = fd.read().strip()
+        else:
+            raise ValueError(f"Unable to find 'latest' file at {latest_path}")
+
+    ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
+
+    if not os.path.isdir(ds_checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
+
+    return _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
+
+
+def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, output_file, tag=None, exclude_frozen_parameters=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
+    loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``output_file``: path to the pytorch fp32 state_dict output file (e.g. path/pytorch_model.bin)
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+    """
+
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag, exclude_frozen_parameters)
+    print(f"Saving fp32 state dict to {output_file}")
+    torch.save(state_dict, output_file)
+
+
+def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
+    """
+    1. Put the provided model to cpu
+    2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
+    3. Load it into the provided model
+
+    Args:
+        - ``model``: the model object to update
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+
+    Returns:
+        - ``model`: modified model
+
+    Make sure you have plenty of CPU memory available before you call this function. If you don't
+    have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
+    conveniently placed for you in the checkpoint folder.
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
+        model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
+        # submit to model hub or save the model to share with others
+
+    Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
+    of the same application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    """
+    logger.info(f"Extracting fp32 weights")
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+
+    logger.info(f"Overwriting model with fp32 weights")
+    model = model.cpu()
+    model.load_state_dict(state_dict, strict=False)
+
+    return model
+
+
+if __name__ == "__main__":
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("checkpoint_dir",
+                        type=str,
+                        help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+    parser.add_argument(
+        "output_file",
+        type=str,
+        help="path to the pytorch fp32 state_dict output file (e.g. path/checkpoint-12/pytorch_model.bin)")
+    parser.add_argument("-t",
+                        "--tag",
+                        type=str,
+                        default=None,
+                        help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
+    parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+    parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
+    args = parser.parse_args()
+
+    debug = args.debug
+
+    convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
+                                               args.output_file,
+                                               tag=args.tag,
+                                               exclude_frozen_parameters=args.exclude_frozen_parameters)
diff --git a/checkpoint-327/README.md b/checkpoint-327/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..f4a3934800eeb082a0cb833d7b6af4f68eed3615
--- /dev/null
+++ b/checkpoint-327/README.md
@@ -0,0 +1,202 @@
+---
+base_model: nvidia/Llama-3_3-Nemotron-Super-49B-v1
+library_name: peft
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.15.0
\ No newline at end of file
diff --git a/checkpoint-327/adapter_config.json b/checkpoint-327/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..d7d7c196589a89c5b284ba682f27212262154f44
--- /dev/null
+++ b/checkpoint-327/adapter_config.json
@@ -0,0 +1,42 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "nvidia/Llama-3_3-Nemotron-Super-49B-v1",
+  "bias": "none",
+  "corda_config": null,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": null,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 512,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": [
+    "embed_tokens",
+    "lm_head"
+  ],
+  "peft_type": "LORA",
+  "r": 256,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "o_proj",
+    "gate_proj",
+    "q_proj",
+    "k_proj",
+    "v_proj",
+    "up_proj",
+    "down_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_rslora": false
+}
\ No newline at end of file
diff --git a/checkpoint-327/adapter_model.safetensors b/checkpoint-327/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..f4017cb7553e444ebccc04b1fb6335ef644fc5f2
--- /dev/null
+++ b/checkpoint-327/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d620bd29d88e3647a2bc20da73d7fe5daacdf69ec4f7bc8816822938990f9ca4
+size 9016826528
diff --git a/checkpoint-327/global_step325/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/checkpoint-327/global_step325/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..fdbb96c0025a7d12ff589b1f4323d8dbdd08f94a
--- /dev/null
+++ b/checkpoint-327/global_step325/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0aa6143e3b7567d68ae5bac2900c59b05360971f22acd8951e381ae366e7174f
+size 27050164444
diff --git a/checkpoint-327/global_step325/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/checkpoint-327/global_step325/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b3ad24f8a61d21d96ca330da7e209f7cd70cbf44
--- /dev/null
+++ b/checkpoint-327/global_step325/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1b99cd7827c26ccf545a1ec7dbdbe94f74707dd6f761fbb236820267b5bfc6b4
+size 27050169884
diff --git a/checkpoint-327/global_step325/mp_rank_00_model_states.pt b/checkpoint-327/global_step325/mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ef28f90efdf7c58a7a53b36c2aaafaffc3625f8e
--- /dev/null
+++ b/checkpoint-327/global_step325/mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0d9ab4337f0fac6fc42407ce95f838dbaebdc8ad25d72e549fa1d724037edd72
+size 9776788601
diff --git a/checkpoint-327/latest b/checkpoint-327/latest
new file mode 100644
index 0000000000000000000000000000000000000000..c47faa75cf98a3ebcc5595c2608973a481359579
--- /dev/null
+++ b/checkpoint-327/latest
@@ -0,0 +1 @@
+global_step325
\ No newline at end of file
diff --git a/checkpoint-327/rng_state_0.pth b/checkpoint-327/rng_state_0.pth
new file mode 100644
index 0000000000000000000000000000000000000000..4907d52105a172f53cc0a48dd666caae4ba939cc
--- /dev/null
+++ b/checkpoint-327/rng_state_0.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2183b1ee437e8522807ae949aeccbd2ecf48cec72acf37954502988e8ac403c7
+size 14512
diff --git a/checkpoint-327/rng_state_1.pth b/checkpoint-327/rng_state_1.pth
new file mode 100644
index 0000000000000000000000000000000000000000..773fa37efb900a506ed8286bfc12c0ce6f5aa7e8
--- /dev/null
+++ b/checkpoint-327/rng_state_1.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:969dc41f723773ef09d6e241849358b8c3af28880c6659185ad61f83d16d7dc9
+size 14512
diff --git a/checkpoint-327/scheduler.pt b/checkpoint-327/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..54ce5c2d91f68d866a41b157b59de8240304f50e
--- /dev/null
+++ b/checkpoint-327/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eecf9ae6fa50caf9c774ed15f04010c8de6904c7d2ff6be7c7f21e4e1a0bdec5
+size 1064
diff --git a/checkpoint-327/special_tokens_map.json b/checkpoint-327/special_tokens_map.json
new file mode 100644
index 0000000000000000000000000000000000000000..278b7f0f84be865c4687700ee7b3c63d89a51e18
--- /dev/null
+++ b/checkpoint-327/special_tokens_map.json
@@ -0,0 +1,23 @@
+{
+  "bos_token": {
+    "content": "<|begin_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|eot_id|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|end_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}
diff --git a/checkpoint-327/tokenizer.json b/checkpoint-327/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..1c1d8d5c9024994f1d3b00f9662b8dd89ca13cf2
--- /dev/null
+++ b/checkpoint-327/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920
diff --git a/checkpoint-327/tokenizer_config.json b/checkpoint-327/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..edd01b980c1db496ea102a51c972ee8f5d1a2c74
--- /dev/null
+++ b/checkpoint-327/tokenizer_config.json
@@ -0,0 +1,2064 @@
+{
+  "added_tokens_decoder": {
+    "128000": {
+      "content": "<|begin_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128001": {
+      "content": "<|end_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128002": {
+      "content": "<|reserved_special_token_0|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128003": {
+      "content": "<|reserved_special_token_1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128004": {
+      "content": "<|finetune_right_pad_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128005": {
+      "content": "<|reserved_special_token_2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128006": {
+      "content": "<|start_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128007": {
+      "content": "<|end_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128008": {
+      "content": "<|eom_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128009": {
+      "content": "<|eot_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128010": {
+      "content": "<|python_tag|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128011": {
+      "content": "<|reserved_special_token_3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128012": {
+      "content": "<|reserved_special_token_4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128013": {
+      "content": "<|reserved_special_token_5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128014": {
+      "content": "<|reserved_special_token_6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128015": {
+      "content": "<|reserved_special_token_7|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128016": {
+      "content": "<|reserved_special_token_8|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128017": {
+      "content": "<|reserved_special_token_9|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128018": {
+      "content": "<|reserved_special_token_10|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128019": {
+      "content": "<|reserved_special_token_11|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128020": {
+      "content": "<|reserved_special_token_12|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128021": {
+      "content": "<|reserved_special_token_13|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128022": {
+      "content": "<|reserved_special_token_14|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128023": {
+      "content": "<|reserved_special_token_15|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128024": {
+      "content": "<|reserved_special_token_16|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128025": {
+      "content": "<|reserved_special_token_17|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128026": {
+      "content": "<|reserved_special_token_18|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128027": {
+      "content": "<|reserved_special_token_19|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128028": {
+      "content": "<|reserved_special_token_20|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128029": {
+      "content": "<|reserved_special_token_21|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128030": {
+      "content": "<|reserved_special_token_22|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128031": {
+      "content": "<|reserved_special_token_23|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128032": {
+      "content": "<|reserved_special_token_24|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128033": {
+      "content": "<|reserved_special_token_25|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128034": {
+      "content": "<|reserved_special_token_26|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128035": {
+      "content": "<|reserved_special_token_27|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128036": {
+      "content": "<|reserved_special_token_28|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128037": {
+      "content": "<|reserved_special_token_29|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128038": {
+      "content": "<|reserved_special_token_30|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128039": {
+      "content": "<|reserved_special_token_31|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128040": {
+      "content": "<|reserved_special_token_32|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128041": {
+      "content": "<|reserved_special_token_33|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128042": {
+      "content": "<|reserved_special_token_34|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128043": {
+      "content": "<|reserved_special_token_35|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128044": {
+      "content": "<|reserved_special_token_36|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128045": {
+      "content": "<|reserved_special_token_37|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128046": {
+      "content": "<|reserved_special_token_38|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128047": {
+      "content": "<|reserved_special_token_39|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128048": {
+      "content": "<|reserved_special_token_40|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128049": {
+      "content": "<|reserved_special_token_41|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128050": {
+      "content": "<|reserved_special_token_42|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128051": {
+      "content": "<|reserved_special_token_43|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128052": {
+      "content": "<|reserved_special_token_44|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128053": {
+      "content": "<|reserved_special_token_45|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128054": {
+      "content": "<|reserved_special_token_46|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128055": {
+      "content": "<|reserved_special_token_47|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128056": {
+      "content": "<|reserved_special_token_48|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128057": {
+      "content": "<|reserved_special_token_49|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128058": {
+      "content": "<|reserved_special_token_50|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128059": {
+      "content": "<|reserved_special_token_51|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128060": {
+      "content": "<|reserved_special_token_52|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128061": {
+      "content": "<|reserved_special_token_53|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128062": {
+      "content": "<|reserved_special_token_54|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128063": {
+      "content": "<|reserved_special_token_55|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128064": {
+      "content": "<|reserved_special_token_56|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128065": {
+      "content": "<|reserved_special_token_57|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128066": {
+      "content": "<|reserved_special_token_58|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128067": {
+      "content": "<|reserved_special_token_59|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128068": {
+      "content": "<|reserved_special_token_60|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128069": {
+      "content": "<|reserved_special_token_61|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128070": {
+      "content": "<|reserved_special_token_62|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128071": {
+      "content": "<|reserved_special_token_63|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128072": {
+      "content": "<|reserved_special_token_64|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128073": {
+      "content": "<|reserved_special_token_65|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128074": {
+      "content": "<|reserved_special_token_66|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128075": {
+      "content": "<|reserved_special_token_67|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128076": {
+      "content": "<|reserved_special_token_68|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128077": {
+      "content": "<|reserved_special_token_69|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128078": {
+      "content": "<|reserved_special_token_70|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128079": {
+      "content": "<|reserved_special_token_71|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128080": {
+      "content": "<|reserved_special_token_72|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128081": {
+      "content": "<|reserved_special_token_73|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128082": {
+      "content": "<|reserved_special_token_74|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128083": {
+      "content": "<|reserved_special_token_75|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128084": {
+      "content": "<|reserved_special_token_76|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128085": {
+      "content": "<|reserved_special_token_77|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128086": {
+      "content": "<|reserved_special_token_78|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128087": {
+      "content": "<|reserved_special_token_79|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128088": {
+      "content": "<|reserved_special_token_80|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128089": {
+      "content": "<|reserved_special_token_81|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128090": {
+      "content": "<|reserved_special_token_82|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128091": {
+      "content": "<|reserved_special_token_83|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128092": {
+      "content": "<|reserved_special_token_84|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128093": {
+      "content": "<|reserved_special_token_85|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128094": {
+      "content": "<|reserved_special_token_86|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128095": {
+      "content": "<|reserved_special_token_87|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128096": {
+      "content": "<|reserved_special_token_88|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128097": {
+      "content": "<|reserved_special_token_89|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128098": {
+      "content": "<|reserved_special_token_90|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128099": {
+      "content": "<|reserved_special_token_91|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128100": {
+      "content": "<|reserved_special_token_92|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128101": {
+      "content": "<|reserved_special_token_93|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128102": {
+      "content": "<|reserved_special_token_94|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128103": {
+      "content": "<|reserved_special_token_95|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128104": {
+      "content": "<|reserved_special_token_96|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128105": {
+      "content": "<|reserved_special_token_97|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128106": {
+      "content": "<|reserved_special_token_98|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128107": {
+      "content": "<|reserved_special_token_99|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128108": {
+      "content": "<|reserved_special_token_100|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128109": {
+      "content": "<|reserved_special_token_101|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128110": {
+      "content": "<|reserved_special_token_102|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128111": {
+      "content": "<|reserved_special_token_103|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128112": {
+      "content": "<|reserved_special_token_104|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128113": {
+      "content": "<|reserved_special_token_105|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128114": {
+      "content": "<|reserved_special_token_106|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128115": {
+      "content": "<|reserved_special_token_107|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128116": {
+      "content": "<|reserved_special_token_108|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128117": {
+      "content": "<|reserved_special_token_109|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128118": {
+      "content": "<|reserved_special_token_110|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128119": {
+      "content": "<|reserved_special_token_111|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128120": {
+      "content": "<|reserved_special_token_112|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128121": {
+      "content": "<|reserved_special_token_113|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128122": {
+      "content": "<|reserved_special_token_114|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128123": {
+      "content": "<|reserved_special_token_115|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128124": {
+      "content": "<|reserved_special_token_116|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128125": {
+      "content": "<|reserved_special_token_117|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128126": {
+      "content": "<|reserved_special_token_118|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128127": {
+      "content": "<|reserved_special_token_119|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128128": {
+      "content": "<|reserved_special_token_120|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128129": {
+      "content": "<|reserved_special_token_121|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128130": {
+      "content": "<|reserved_special_token_122|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128131": {
+      "content": "<|reserved_special_token_123|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128132": {
+      "content": "<|reserved_special_token_124|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128133": {
+      "content": "<|reserved_special_token_125|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128134": {
+      "content": "<|reserved_special_token_126|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128135": {
+      "content": "<|reserved_special_token_127|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128136": {
+      "content": "<|reserved_special_token_128|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128137": {
+      "content": "<|reserved_special_token_129|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128138": {
+      "content": "<|reserved_special_token_130|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128139": {
+      "content": "<|reserved_special_token_131|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128140": {
+      "content": "<|reserved_special_token_132|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128141": {
+      "content": "<|reserved_special_token_133|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128142": {
+      "content": "<|reserved_special_token_134|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128143": {
+      "content": "<|reserved_special_token_135|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128144": {
+      "content": "<|reserved_special_token_136|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128145": {
+      "content": "<|reserved_special_token_137|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128146": {
+      "content": "<|reserved_special_token_138|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128147": {
+      "content": "<|reserved_special_token_139|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128148": {
+      "content": "<|reserved_special_token_140|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128149": {
+      "content": "<|reserved_special_token_141|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128150": {
+      "content": "<|reserved_special_token_142|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128151": {
+      "content": "<|reserved_special_token_143|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128152": {
+      "content": "<|reserved_special_token_144|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128153": {
+      "content": "<|reserved_special_token_145|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128154": {
+      "content": "<|reserved_special_token_146|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128155": {
+      "content": "<|reserved_special_token_147|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128156": {
+      "content": "<|reserved_special_token_148|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128157": {
+      "content": "<|reserved_special_token_149|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128158": {
+      "content": "<|reserved_special_token_150|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128159": {
+      "content": "<|reserved_special_token_151|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128160": {
+      "content": "<|reserved_special_token_152|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128161": {
+      "content": "<|reserved_special_token_153|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128162": {
+      "content": "<|reserved_special_token_154|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128163": {
+      "content": "<|reserved_special_token_155|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128164": {
+      "content": "<|reserved_special_token_156|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128165": {
+      "content": "<|reserved_special_token_157|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128166": {
+      "content": "<|reserved_special_token_158|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128167": {
+      "content": "<|reserved_special_token_159|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128168": {
+      "content": "<|reserved_special_token_160|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128169": {
+      "content": "<|reserved_special_token_161|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128170": {
+      "content": "<|reserved_special_token_162|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128171": {
+      "content": "<|reserved_special_token_163|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128172": {
+      "content": "<|reserved_special_token_164|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128173": {
+      "content": "<|reserved_special_token_165|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128174": {
+      "content": "<|reserved_special_token_166|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128175": {
+      "content": "<|reserved_special_token_167|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128176": {
+      "content": "<|reserved_special_token_168|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128177": {
+      "content": "<|reserved_special_token_169|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128178": {
+      "content": "<|reserved_special_token_170|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128179": {
+      "content": "<|reserved_special_token_171|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128180": {
+      "content": "<|reserved_special_token_172|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128181": {
+      "content": "<|reserved_special_token_173|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128182": {
+      "content": "<|reserved_special_token_174|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128183": {
+      "content": "<|reserved_special_token_175|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128184": {
+      "content": "<|reserved_special_token_176|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128185": {
+      "content": "<|reserved_special_token_177|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128186": {
+      "content": "<|reserved_special_token_178|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128187": {
+      "content": "<|reserved_special_token_179|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128188": {
+      "content": "<|reserved_special_token_180|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128189": {
+      "content": "<|reserved_special_token_181|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128190": {
+      "content": "<|reserved_special_token_182|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128191": {
+      "content": "<|reserved_special_token_183|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128192": {
+      "content": "<|reserved_special_token_184|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128193": {
+      "content": "<|reserved_special_token_185|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128194": {
+      "content": "<|reserved_special_token_186|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128195": {
+      "content": "<|reserved_special_token_187|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128196": {
+      "content": "<|reserved_special_token_188|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128197": {
+      "content": "<|reserved_special_token_189|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128198": {
+      "content": "<|reserved_special_token_190|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128199": {
+      "content": "<|reserved_special_token_191|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128200": {
+      "content": "<|reserved_special_token_192|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128201": {
+      "content": "<|reserved_special_token_193|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128202": {
+      "content": "<|reserved_special_token_194|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128203": {
+      "content": "<|reserved_special_token_195|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128204": {
+      "content": "<|reserved_special_token_196|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128205": {
+      "content": "<|reserved_special_token_197|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128206": {
+      "content": "<|reserved_special_token_198|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128207": {
+      "content": "<|reserved_special_token_199|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128208": {
+      "content": "<|reserved_special_token_200|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128209": {
+      "content": "<|reserved_special_token_201|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128210": {
+      "content": "<|reserved_special_token_202|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128211": {
+      "content": "<|reserved_special_token_203|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128212": {
+      "content": "<|reserved_special_token_204|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128213": {
+      "content": "<|reserved_special_token_205|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128214": {
+      "content": "<|reserved_special_token_206|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128215": {
+      "content": "<|reserved_special_token_207|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128216": {
+      "content": "<|reserved_special_token_208|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128217": {
+      "content": "<|reserved_special_token_209|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128218": {
+      "content": "<|reserved_special_token_210|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128219": {
+      "content": "<|reserved_special_token_211|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128220": {
+      "content": "<|reserved_special_token_212|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128221": {
+      "content": "<|reserved_special_token_213|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128222": {
+      "content": "<|reserved_special_token_214|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128223": {
+      "content": "<|reserved_special_token_215|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128224": {
+      "content": "<|reserved_special_token_216|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128225": {
+      "content": "<|reserved_special_token_217|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128226": {
+      "content": "<|reserved_special_token_218|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128227": {
+      "content": "<|reserved_special_token_219|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128228": {
+      "content": "<|reserved_special_token_220|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128229": {
+      "content": "<|reserved_special_token_221|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128230": {
+      "content": "<|reserved_special_token_222|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128231": {
+      "content": "<|reserved_special_token_223|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128232": {
+      "content": "<|reserved_special_token_224|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128233": {
+      "content": "<|reserved_special_token_225|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128234": {
+      "content": "<|reserved_special_token_226|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128235": {
+      "content": "<|reserved_special_token_227|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128236": {
+      "content": "<|reserved_special_token_228|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128237": {
+      "content": "<|reserved_special_token_229|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128238": {
+      "content": "<|reserved_special_token_230|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128239": {
+      "content": "<|reserved_special_token_231|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128240": {
+      "content": "<|reserved_special_token_232|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128241": {
+      "content": "<|reserved_special_token_233|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128242": {
+      "content": "<|reserved_special_token_234|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128243": {
+      "content": "<|reserved_special_token_235|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128244": {
+      "content": "<|reserved_special_token_236|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128245": {
+      "content": "<|reserved_special_token_237|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128246": {
+      "content": "<|reserved_special_token_238|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128247": {
+      "content": "<|reserved_special_token_239|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128248": {
+      "content": "<|reserved_special_token_240|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128249": {
+      "content": "<|reserved_special_token_241|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128250": {
+      "content": "<|reserved_special_token_242|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128251": {
+      "content": "<|reserved_special_token_243|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128252": {
+      "content": "<|reserved_special_token_244|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128253": {
+      "content": "<|reserved_special_token_245|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128254": {
+      "content": "<|reserved_special_token_246|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128255": {
+      "content": "<|reserved_special_token_247|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|begin_of_text|>",
+  "chat_template": "{{- bos_token }}{%- if messages[0]['role'] == 'system' %}{%- set system_message = messages[0]['content']|trim %}{%- set messages = messages[1:] %}{%- else %}{%- set system_message = \"\" %}{%- endif %}{{- \"<|start_header_id|>system<|end_header_id|>\\n\\n\" }}{{- system_message }}{{- \"<|eot_id|>\" }}{%- for message in messages %}{%- if message['role'] == 'assistant' and '</think>' in message['content'] %}{%- set content = message['content'].split('</think>')[-1].lstrip() %}{%- else %}{%- set content = message['content'] %}{%- endif %}{{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\\n\\n' + content | trim + '<|eot_id|>' }}{%- endfor %}{%- if add_generation_prompt %}{{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' }}{%- endif %}",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|eot_id|>",
+  "extra_special_tokens": {},
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "pad_token": "<|end_of_text|>",
+  "tokenizer_class": "PreTrainedTokenizer"
+}
diff --git a/checkpoint-327/trainer_state.json b/checkpoint-327/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..553c2d5701368bfe6294c20c5b0d741a0f189a88
--- /dev/null
+++ b/checkpoint-327/trainer_state.json
@@ -0,0 +1,2322 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.9786585365853657,
+  "eval_steps": 500,
+  "global_step": 327,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.009146341463414634,
+      "grad_norm": 40.05127716064453,
+      "learning_rate": 5.0000000000000004e-08,
+      "loss": 5.1282,
+      "step": 1
+    },
+    {
+      "epoch": 0.018292682926829267,
+      "grad_norm": 39.01039505004883,
+      "learning_rate": 1.0000000000000001e-07,
+      "loss": 5.0586,
+      "step": 2
+    },
+    {
+      "epoch": 0.027439024390243903,
+      "grad_norm": 40.20766067504883,
+      "learning_rate": 1.5000000000000002e-07,
+      "loss": 5.0311,
+      "step": 3
+    },
+    {
+      "epoch": 0.036585365853658534,
+      "grad_norm": 38.70350646972656,
+      "learning_rate": 2.0000000000000002e-07,
+      "loss": 5.1598,
+      "step": 4
+    },
+    {
+      "epoch": 0.04573170731707317,
+      "grad_norm": 39.230979919433594,
+      "learning_rate": 2.5000000000000004e-07,
+      "loss": 5.0812,
+      "step": 5
+    },
+    {
+      "epoch": 0.054878048780487805,
+      "grad_norm": 39.45293045043945,
+      "learning_rate": 3.0000000000000004e-07,
+      "loss": 5.0841,
+      "step": 6
+    },
+    {
+      "epoch": 0.06402439024390244,
+      "grad_norm": 38.92170333862305,
+      "learning_rate": 3.5000000000000004e-07,
+      "loss": 5.022,
+      "step": 7
+    },
+    {
+      "epoch": 0.07317073170731707,
+      "grad_norm": 38.932369232177734,
+      "learning_rate": 4.0000000000000003e-07,
+      "loss": 5.0876,
+      "step": 8
+    },
+    {
+      "epoch": 0.08231707317073171,
+      "grad_norm": 39.60245132446289,
+      "learning_rate": 4.5000000000000003e-07,
+      "loss": 5.0798,
+      "step": 9
+    },
+    {
+      "epoch": 0.09146341463414634,
+      "grad_norm": 39.53607177734375,
+      "learning_rate": 5.000000000000001e-07,
+      "loss": 5.0105,
+      "step": 10
+    },
+    {
+      "epoch": 0.10060975609756098,
+      "grad_norm": 39.50700378417969,
+      "learning_rate": 5.5e-07,
+      "loss": 4.9951,
+      "step": 11
+    },
+    {
+      "epoch": 0.10975609756097561,
+      "grad_norm": 39.52376174926758,
+      "learning_rate": 6.000000000000001e-07,
+      "loss": 4.9163,
+      "step": 12
+    },
+    {
+      "epoch": 0.11890243902439024,
+      "grad_norm": 39.15214157104492,
+      "learning_rate": 6.5e-07,
+      "loss": 4.9634,
+      "step": 13
+    },
+    {
+      "epoch": 0.12804878048780488,
+      "grad_norm": 37.795066833496094,
+      "learning_rate": 7.000000000000001e-07,
+      "loss": 4.819,
+      "step": 14
+    },
+    {
+      "epoch": 0.13719512195121952,
+      "grad_norm": 38.23014450073242,
+      "learning_rate": 7.5e-07,
+      "loss": 4.8756,
+      "step": 15
+    },
+    {
+      "epoch": 0.14634146341463414,
+      "grad_norm": 38.57551574707031,
+      "learning_rate": 8.000000000000001e-07,
+      "loss": 4.81,
+      "step": 16
+    },
+    {
+      "epoch": 0.15548780487804878,
+      "grad_norm": 36.06121063232422,
+      "learning_rate": 8.500000000000001e-07,
+      "loss": 4.7901,
+      "step": 17
+    },
+    {
+      "epoch": 0.16463414634146342,
+      "grad_norm": 34.38008117675781,
+      "learning_rate": 9.000000000000001e-07,
+      "loss": 4.5775,
+      "step": 18
+    },
+    {
+      "epoch": 0.17378048780487804,
+      "grad_norm": 32.60890579223633,
+      "learning_rate": 9.500000000000001e-07,
+      "loss": 4.3737,
+      "step": 19
+    },
+    {
+      "epoch": 0.18292682926829268,
+      "grad_norm": 31.983110427856445,
+      "learning_rate": 1.0000000000000002e-06,
+      "loss": 4.3773,
+      "step": 20
+    },
+    {
+      "epoch": 0.19207317073170732,
+      "grad_norm": 27.69571876525879,
+      "learning_rate": 1.0500000000000001e-06,
+      "loss": 4.1276,
+      "step": 21
+    },
+    {
+      "epoch": 0.20121951219512196,
+      "grad_norm": 26.542314529418945,
+      "learning_rate": 1.1e-06,
+      "loss": 4.0641,
+      "step": 22
+    },
+    {
+      "epoch": 0.21036585365853658,
+      "grad_norm": 23.26603126525879,
+      "learning_rate": 1.1500000000000002e-06,
+      "loss": 4.0197,
+      "step": 23
+    },
+    {
+      "epoch": 0.21951219512195122,
+      "grad_norm": 21.04385757446289,
+      "learning_rate": 1.2000000000000002e-06,
+      "loss": 4.0179,
+      "step": 24
+    },
+    {
+      "epoch": 0.22865853658536586,
+      "grad_norm": 19.790103912353516,
+      "learning_rate": 1.25e-06,
+      "loss": 3.8334,
+      "step": 25
+    },
+    {
+      "epoch": 0.23780487804878048,
+      "grad_norm": 18.2396183013916,
+      "learning_rate": 1.3e-06,
+      "loss": 3.7027,
+      "step": 26
+    },
+    {
+      "epoch": 0.24695121951219512,
+      "grad_norm": 18.725860595703125,
+      "learning_rate": 1.3500000000000002e-06,
+      "loss": 3.7268,
+      "step": 27
+    },
+    {
+      "epoch": 0.25609756097560976,
+      "grad_norm": 18.91561508178711,
+      "learning_rate": 1.4000000000000001e-06,
+      "loss": 3.52,
+      "step": 28
+    },
+    {
+      "epoch": 0.2652439024390244,
+      "grad_norm": 18.196651458740234,
+      "learning_rate": 1.45e-06,
+      "loss": 3.4765,
+      "step": 29
+    },
+    {
+      "epoch": 0.27439024390243905,
+      "grad_norm": 17.778606414794922,
+      "learning_rate": 1.5e-06,
+      "loss": 3.3573,
+      "step": 30
+    },
+    {
+      "epoch": 0.28353658536585363,
+      "grad_norm": 16.23012924194336,
+      "learning_rate": 1.5500000000000002e-06,
+      "loss": 3.2783,
+      "step": 31
+    },
+    {
+      "epoch": 0.2926829268292683,
+      "grad_norm": 15.836565017700195,
+      "learning_rate": 1.6000000000000001e-06,
+      "loss": 3.2319,
+      "step": 32
+    },
+    {
+      "epoch": 0.3018292682926829,
+      "grad_norm": 11.919289588928223,
+      "learning_rate": 1.6500000000000003e-06,
+      "loss": 3.1345,
+      "step": 33
+    },
+    {
+      "epoch": 0.31097560975609756,
+      "grad_norm": 11.086843490600586,
+      "learning_rate": 1.7000000000000002e-06,
+      "loss": 3.1153,
+      "step": 34
+    },
+    {
+      "epoch": 0.3201219512195122,
+      "grad_norm": 7.563486576080322,
+      "learning_rate": 1.75e-06,
+      "loss": 2.9784,
+      "step": 35
+    },
+    {
+      "epoch": 0.32926829268292684,
+      "grad_norm": 7.60545539855957,
+      "learning_rate": 1.8000000000000001e-06,
+      "loss": 2.8568,
+      "step": 36
+    },
+    {
+      "epoch": 0.3384146341463415,
+      "grad_norm": 7.071218013763428,
+      "learning_rate": 1.85e-06,
+      "loss": 2.8879,
+      "step": 37
+    },
+    {
+      "epoch": 0.3475609756097561,
+      "grad_norm": 8.200672149658203,
+      "learning_rate": 1.9000000000000002e-06,
+      "loss": 2.7437,
+      "step": 38
+    },
+    {
+      "epoch": 0.3567073170731707,
+      "grad_norm": 12.079374313354492,
+      "learning_rate": 1.9500000000000004e-06,
+      "loss": 2.7589,
+      "step": 39
+    },
+    {
+      "epoch": 0.36585365853658536,
+      "grad_norm": 12.21938705444336,
+      "learning_rate": 2.0000000000000003e-06,
+      "loss": 2.7098,
+      "step": 40
+    },
+    {
+      "epoch": 0.375,
+      "grad_norm": 10.224228858947754,
+      "learning_rate": 2.05e-06,
+      "loss": 2.6803,
+      "step": 41
+    },
+    {
+      "epoch": 0.38414634146341464,
+      "grad_norm": 10.054616928100586,
+      "learning_rate": 2.1000000000000002e-06,
+      "loss": 2.4658,
+      "step": 42
+    },
+    {
+      "epoch": 0.3932926829268293,
+      "grad_norm": 8.074049949645996,
+      "learning_rate": 2.15e-06,
+      "loss": 2.6341,
+      "step": 43
+    },
+    {
+      "epoch": 0.4024390243902439,
+      "grad_norm": 5.721987724304199,
+      "learning_rate": 2.2e-06,
+      "loss": 2.3848,
+      "step": 44
+    },
+    {
+      "epoch": 0.4115853658536585,
+      "grad_norm": 4.646566390991211,
+      "learning_rate": 2.25e-06,
+      "loss": 2.4323,
+      "step": 45
+    },
+    {
+      "epoch": 0.42073170731707316,
+      "grad_norm": 4.591104984283447,
+      "learning_rate": 2.3000000000000004e-06,
+      "loss": 2.4022,
+      "step": 46
+    },
+    {
+      "epoch": 0.4298780487804878,
+      "grad_norm": 4.822715759277344,
+      "learning_rate": 2.35e-06,
+      "loss": 2.2768,
+      "step": 47
+    },
+    {
+      "epoch": 0.43902439024390244,
+      "grad_norm": 5.176568984985352,
+      "learning_rate": 2.4000000000000003e-06,
+      "loss": 2.281,
+      "step": 48
+    },
+    {
+      "epoch": 0.4481707317073171,
+      "grad_norm": 4.731242656707764,
+      "learning_rate": 2.4500000000000003e-06,
+      "loss": 2.235,
+      "step": 49
+    },
+    {
+      "epoch": 0.4573170731707317,
+      "grad_norm": 4.6343865394592285,
+      "learning_rate": 2.5e-06,
+      "loss": 2.1465,
+      "step": 50
+    },
+    {
+      "epoch": 0.46646341463414637,
+      "grad_norm": 4.669382095336914,
+      "learning_rate": 2.55e-06,
+      "loss": 2.0969,
+      "step": 51
+    },
+    {
+      "epoch": 0.47560975609756095,
+      "grad_norm": 4.207769870758057,
+      "learning_rate": 2.6e-06,
+      "loss": 2.0308,
+      "step": 52
+    },
+    {
+      "epoch": 0.4847560975609756,
+      "grad_norm": 5.222470760345459,
+      "learning_rate": 2.6500000000000005e-06,
+      "loss": 1.9445,
+      "step": 53
+    },
+    {
+      "epoch": 0.49390243902439024,
+      "grad_norm": 5.10925817489624,
+      "learning_rate": 2.7000000000000004e-06,
+      "loss": 1.7763,
+      "step": 54
+    },
+    {
+      "epoch": 0.5030487804878049,
+      "grad_norm": 4.8151044845581055,
+      "learning_rate": 2.7500000000000004e-06,
+      "loss": 1.7661,
+      "step": 55
+    },
+    {
+      "epoch": 0.5121951219512195,
+      "grad_norm": 6.155700206756592,
+      "learning_rate": 2.8000000000000003e-06,
+      "loss": 1.6665,
+      "step": 56
+    },
+    {
+      "epoch": 0.5213414634146342,
+      "grad_norm": 4.719021320343018,
+      "learning_rate": 2.85e-06,
+      "loss": 1.5532,
+      "step": 57
+    },
+    {
+      "epoch": 0.5304878048780488,
+      "grad_norm": 4.591799259185791,
+      "learning_rate": 2.9e-06,
+      "loss": 1.5244,
+      "step": 58
+    },
+    {
+      "epoch": 0.5396341463414634,
+      "grad_norm": 5.483795642852783,
+      "learning_rate": 2.95e-06,
+      "loss": 1.3816,
+      "step": 59
+    },
+    {
+      "epoch": 0.5487804878048781,
+      "grad_norm": 4.32553768157959,
+      "learning_rate": 3e-06,
+      "loss": 1.4268,
+      "step": 60
+    },
+    {
+      "epoch": 0.5579268292682927,
+      "grad_norm": 8.296231269836426,
+      "learning_rate": 3.05e-06,
+      "loss": 1.2887,
+      "step": 61
+    },
+    {
+      "epoch": 0.5670731707317073,
+      "grad_norm": 5.07594108581543,
+      "learning_rate": 3.1000000000000004e-06,
+      "loss": 1.228,
+      "step": 62
+    },
+    {
+      "epoch": 0.5762195121951219,
+      "grad_norm": 8.427435874938965,
+      "learning_rate": 3.1500000000000003e-06,
+      "loss": 1.1724,
+      "step": 63
+    },
+    {
+      "epoch": 0.5853658536585366,
+      "grad_norm": 22.45092010498047,
+      "learning_rate": 3.2000000000000003e-06,
+      "loss": 1.0871,
+      "step": 64
+    },
+    {
+      "epoch": 0.5945121951219512,
+      "grad_norm": 20.67412567138672,
+      "learning_rate": 3.2500000000000002e-06,
+      "loss": 1.0108,
+      "step": 65
+    },
+    {
+      "epoch": 0.6036585365853658,
+      "grad_norm": 4.921634674072266,
+      "learning_rate": 3.3000000000000006e-06,
+      "loss": 0.9775,
+      "step": 66
+    },
+    {
+      "epoch": 0.6128048780487805,
+      "grad_norm": 4.477286338806152,
+      "learning_rate": 3.3500000000000005e-06,
+      "loss": 0.9554,
+      "step": 67
+    },
+    {
+      "epoch": 0.6219512195121951,
+      "grad_norm": 8.190194129943848,
+      "learning_rate": 3.4000000000000005e-06,
+      "loss": 0.8583,
+      "step": 68
+    },
+    {
+      "epoch": 0.6310975609756098,
+      "grad_norm": 29.21201515197754,
+      "learning_rate": 3.45e-06,
+      "loss": 0.8499,
+      "step": 69
+    },
+    {
+      "epoch": 0.6402439024390244,
+      "grad_norm": 33.81047439575195,
+      "learning_rate": 3.5e-06,
+      "loss": 0.858,
+      "step": 70
+    },
+    {
+      "epoch": 0.649390243902439,
+      "grad_norm": 3.5125606060028076,
+      "learning_rate": 3.5500000000000003e-06,
+      "loss": 0.7227,
+      "step": 71
+    },
+    {
+      "epoch": 0.6585365853658537,
+      "grad_norm": 18.493423461914062,
+      "learning_rate": 3.6000000000000003e-06,
+      "loss": 0.7447,
+      "step": 72
+    },
+    {
+      "epoch": 0.6676829268292683,
+      "grad_norm": 24.923927307128906,
+      "learning_rate": 3.65e-06,
+      "loss": 0.8423,
+      "step": 73
+    },
+    {
+      "epoch": 0.676829268292683,
+      "grad_norm": 9.870631217956543,
+      "learning_rate": 3.7e-06,
+      "loss": 0.6934,
+      "step": 74
+    },
+    {
+      "epoch": 0.6859756097560976,
+      "grad_norm": 20.598569869995117,
+      "learning_rate": 3.7500000000000005e-06,
+      "loss": 0.7005,
+      "step": 75
+    },
+    {
+      "epoch": 0.6951219512195121,
+      "grad_norm": 30.662466049194336,
+      "learning_rate": 3.8000000000000005e-06,
+      "loss": 0.6226,
+      "step": 76
+    },
+    {
+      "epoch": 0.7042682926829268,
+      "grad_norm": 28.930774688720703,
+      "learning_rate": 3.85e-06,
+      "loss": 0.6357,
+      "step": 77
+    },
+    {
+      "epoch": 0.7134146341463414,
+      "grad_norm": 7.140467166900635,
+      "learning_rate": 3.900000000000001e-06,
+      "loss": 0.6129,
+      "step": 78
+    },
+    {
+      "epoch": 0.7225609756097561,
+      "grad_norm": 6.322911739349365,
+      "learning_rate": 3.95e-06,
+      "loss": 0.6086,
+      "step": 79
+    },
+    {
+      "epoch": 0.7317073170731707,
+      "grad_norm": 5.404141426086426,
+      "learning_rate": 4.000000000000001e-06,
+      "loss": 0.5986,
+      "step": 80
+    },
+    {
+      "epoch": 0.7408536585365854,
+      "grad_norm": 24.445228576660156,
+      "learning_rate": 4.05e-06,
+      "loss": 0.5932,
+      "step": 81
+    },
+    {
+      "epoch": 0.75,
+      "grad_norm": 14.275381088256836,
+      "learning_rate": 4.1e-06,
+      "loss": 0.5286,
+      "step": 82
+    },
+    {
+      "epoch": 0.7591463414634146,
+      "grad_norm": 9.77132511138916,
+      "learning_rate": 4.15e-06,
+      "loss": 0.4814,
+      "step": 83
+    },
+    {
+      "epoch": 0.7682926829268293,
+      "grad_norm": 11.043930053710938,
+      "learning_rate": 4.2000000000000004e-06,
+      "loss": 0.4953,
+      "step": 84
+    },
+    {
+      "epoch": 0.7774390243902439,
+      "grad_norm": 3.1333775520324707,
+      "learning_rate": 4.25e-06,
+      "loss": 0.4794,
+      "step": 85
+    },
+    {
+      "epoch": 0.7865853658536586,
+      "grad_norm": 9.50448989868164,
+      "learning_rate": 4.3e-06,
+      "loss": 0.4451,
+      "step": 86
+    },
+    {
+      "epoch": 0.7957317073170732,
+      "grad_norm": 5.867074489593506,
+      "learning_rate": 4.350000000000001e-06,
+      "loss": 0.4224,
+      "step": 87
+    },
+    {
+      "epoch": 0.8048780487804879,
+      "grad_norm": 12.556106567382812,
+      "learning_rate": 4.4e-06,
+      "loss": 0.4224,
+      "step": 88
+    },
+    {
+      "epoch": 0.8140243902439024,
+      "grad_norm": 7.916622638702393,
+      "learning_rate": 4.450000000000001e-06,
+      "loss": 0.4091,
+      "step": 89
+    },
+    {
+      "epoch": 0.823170731707317,
+      "grad_norm": 2.970932722091675,
+      "learning_rate": 4.5e-06,
+      "loss": 0.4181,
+      "step": 90
+    },
+    {
+      "epoch": 0.8323170731707317,
+      "grad_norm": 17.468870162963867,
+      "learning_rate": 4.5500000000000005e-06,
+      "loss": 0.4031,
+      "step": 91
+    },
+    {
+      "epoch": 0.8414634146341463,
+      "grad_norm": 15.861083984375,
+      "learning_rate": 4.600000000000001e-06,
+      "loss": 0.3604,
+      "step": 92
+    },
+    {
+      "epoch": 0.850609756097561,
+      "grad_norm": 6.715958118438721,
+      "learning_rate": 4.65e-06,
+      "loss": 0.3489,
+      "step": 93
+    },
+    {
+      "epoch": 0.8597560975609756,
+      "grad_norm": 5.439844608306885,
+      "learning_rate": 4.7e-06,
+      "loss": 0.3339,
+      "step": 94
+    },
+    {
+      "epoch": 0.8689024390243902,
+      "grad_norm": 4.650434494018555,
+      "learning_rate": 4.75e-06,
+      "loss": 0.3189,
+      "step": 95
+    },
+    {
+      "epoch": 0.8780487804878049,
+      "grad_norm": 15.319528579711914,
+      "learning_rate": 4.800000000000001e-06,
+      "loss": 0.3347,
+      "step": 96
+    },
+    {
+      "epoch": 0.8871951219512195,
+      "grad_norm": 5.151925563812256,
+      "learning_rate": 4.85e-06,
+      "loss": 0.3043,
+      "step": 97
+    },
+    {
+      "epoch": 0.8963414634146342,
+      "grad_norm": 6.351178169250488,
+      "learning_rate": 4.9000000000000005e-06,
+      "loss": 0.2939,
+      "step": 98
+    },
+    {
+      "epoch": 0.9054878048780488,
+      "grad_norm": 3.942911148071289,
+      "learning_rate": 4.95e-06,
+      "loss": 0.2887,
+      "step": 99
+    },
+    {
+      "epoch": 0.9146341463414634,
+      "grad_norm": 3.0885493755340576,
+      "learning_rate": 5e-06,
+      "loss": 0.2992,
+      "step": 100
+    },
+    {
+      "epoch": 0.9237804878048781,
+      "grad_norm": 7.023274898529053,
+      "learning_rate": 4.999959803423605e-06,
+      "loss": 0.2607,
+      "step": 101
+    },
+    {
+      "epoch": 0.9329268292682927,
+      "grad_norm": 8.774216651916504,
+      "learning_rate": 4.9998392149870315e-06,
+      "loss": 0.2674,
+      "step": 102
+    },
+    {
+      "epoch": 0.9420731707317073,
+      "grad_norm": 3.5439791679382324,
+      "learning_rate": 4.999638238568072e-06,
+      "loss": 0.2602,
+      "step": 103
+    },
+    {
+      "epoch": 0.9512195121951219,
+      "grad_norm": 7.669381618499756,
+      "learning_rate": 4.999356880629579e-06,
+      "loss": 0.2718,
+      "step": 104
+    },
+    {
+      "epoch": 0.9603658536585366,
+      "grad_norm": 6.9407854080200195,
+      "learning_rate": 4.998995150219252e-06,
+      "loss": 0.2494,
+      "step": 105
+    },
+    {
+      "epoch": 0.9695121951219512,
+      "grad_norm": 12.144411087036133,
+      "learning_rate": 4.998553058969352e-06,
+      "loss": 0.2297,
+      "step": 106
+    },
+    {
+      "epoch": 0.9786585365853658,
+      "grad_norm": 6.956138610839844,
+      "learning_rate": 4.998030621096322e-06,
+      "loss": 0.2386,
+      "step": 107
+    },
+    {
+      "epoch": 0.9878048780487805,
+      "grad_norm": 8.304037094116211,
+      "learning_rate": 4.997427853400333e-06,
+      "loss": 0.23,
+      "step": 108
+    },
+    {
+      "epoch": 0.9969512195121951,
+      "grad_norm": 7.031365394592285,
+      "learning_rate": 4.996744775264743e-06,
+      "loss": 0.2346,
+      "step": 109
+    },
+    {
+      "epoch": 1.0,
+      "grad_norm": 7.031365394592285,
+      "learning_rate": 4.9959814086554735e-06,
+      "loss": 0.0739,
+      "step": 110
+    },
+    {
+      "epoch": 1.0091463414634145,
+      "grad_norm": 2.084608793258667,
+      "learning_rate": 4.995137778120306e-06,
+      "loss": 0.2422,
+      "step": 111
+    },
+    {
+      "epoch": 1.0182926829268293,
+      "grad_norm": 38.12245178222656,
+      "learning_rate": 4.994213910788085e-06,
+      "loss": 0.1992,
+      "step": 112
+    },
+    {
+      "epoch": 1.0274390243902438,
+      "grad_norm": 2.817565441131592,
+      "learning_rate": 4.993209836367856e-06,
+      "loss": 0.1882,
+      "step": 113
+    },
+    {
+      "epoch": 1.0365853658536586,
+      "grad_norm": 11.774249076843262,
+      "learning_rate": 4.9921255871479e-06,
+      "loss": 0.2235,
+      "step": 114
+    },
+    {
+      "epoch": 1.045731707317073,
+      "grad_norm": 2.3362019062042236,
+      "learning_rate": 4.9909611979947046e-06,
+      "loss": 0.1875,
+      "step": 115
+    },
+    {
+      "epoch": 1.0548780487804879,
+      "grad_norm": 1.9516818523406982,
+      "learning_rate": 4.989716706351835e-06,
+      "loss": 0.1858,
+      "step": 116
+    },
+    {
+      "epoch": 1.0640243902439024,
+      "grad_norm": 1.5484645366668701,
+      "learning_rate": 4.9883921522387336e-06,
+      "loss": 0.1845,
+      "step": 117
+    },
+    {
+      "epoch": 1.0731707317073171,
+      "grad_norm": 7.430396556854248,
+      "learning_rate": 4.986987578249433e-06,
+      "loss": 0.184,
+      "step": 118
+    },
+    {
+      "epoch": 1.0823170731707317,
+      "grad_norm": 2.0725948810577393,
+      "learning_rate": 4.9855030295511845e-06,
+      "loss": 0.177,
+      "step": 119
+    },
+    {
+      "epoch": 1.0914634146341464,
+      "grad_norm": 4.836520671844482,
+      "learning_rate": 4.98393855388301e-06,
+      "loss": 0.1744,
+      "step": 120
+    },
+    {
+      "epoch": 1.100609756097561,
+      "grad_norm": 5.22581672668457,
+      "learning_rate": 4.982294201554162e-06,
+      "loss": 0.1599,
+      "step": 121
+    },
+    {
+      "epoch": 1.1097560975609757,
+      "grad_norm": 5.406208038330078,
+      "learning_rate": 4.980570025442507e-06,
+      "loss": 0.1645,
+      "step": 122
+    },
+    {
+      "epoch": 1.1189024390243902,
+      "grad_norm": 8.736640930175781,
+      "learning_rate": 4.978766080992827e-06,
+      "loss": 0.1943,
+      "step": 123
+    },
+    {
+      "epoch": 1.1280487804878048,
+      "grad_norm": 9.184198379516602,
+      "learning_rate": 4.976882426215035e-06,
+      "loss": 0.1753,
+      "step": 124
+    },
+    {
+      "epoch": 1.1371951219512195,
+      "grad_norm": 5.317800521850586,
+      "learning_rate": 4.974919121682307e-06,
+      "loss": 0.1897,
+      "step": 125
+    },
+    {
+      "epoch": 1.146341463414634,
+      "grad_norm": 11.803364753723145,
+      "learning_rate": 4.972876230529143e-06,
+      "loss": 0.1516,
+      "step": 126
+    },
+    {
+      "epoch": 1.1554878048780488,
+      "grad_norm": 31.74310302734375,
+      "learning_rate": 4.9707538184493255e-06,
+      "loss": 0.1947,
+      "step": 127
+    },
+    {
+      "epoch": 1.1646341463414633,
+      "grad_norm": 1.956932783126831,
+      "learning_rate": 4.968551953693813e-06,
+      "loss": 0.1708,
+      "step": 128
+    },
+    {
+      "epoch": 1.173780487804878,
+      "grad_norm": 4.64445686340332,
+      "learning_rate": 4.966270707068548e-06,
+      "loss": 0.1711,
+      "step": 129
+    },
+    {
+      "epoch": 1.1829268292682926,
+      "grad_norm": 4.442819595336914,
+      "learning_rate": 4.963910151932171e-06,
+      "loss": 0.1572,
+      "step": 130
+    },
+    {
+      "epoch": 1.1920731707317074,
+      "grad_norm": 1.9702759981155396,
+      "learning_rate": 4.961470364193673e-06,
+      "loss": 0.1364,
+      "step": 131
+    },
+    {
+      "epoch": 1.201219512195122,
+      "grad_norm": 7.816587448120117,
+      "learning_rate": 4.9589514223099425e-06,
+      "loss": 0.1624,
+      "step": 132
+    },
+    {
+      "epoch": 1.2103658536585367,
+      "grad_norm": 61.18424606323242,
+      "learning_rate": 4.956353407283252e-06,
+      "loss": 0.26,
+      "step": 133
+    },
+    {
+      "epoch": 1.2195121951219512,
+      "grad_norm": 15.649847984313965,
+      "learning_rate": 4.9536764026586505e-06,
+      "loss": 0.1543,
+      "step": 134
+    },
+    {
+      "epoch": 1.228658536585366,
+      "grad_norm": 2.249363660812378,
+      "learning_rate": 4.950920494521274e-06,
+      "loss": 0.152,
+      "step": 135
+    },
+    {
+      "epoch": 1.2378048780487805,
+      "grad_norm": 4.055661678314209,
+      "learning_rate": 4.948085771493579e-06,
+      "loss": 0.1402,
+      "step": 136
+    },
+    {
+      "epoch": 1.2469512195121952,
+      "grad_norm": 2.2803292274475098,
+      "learning_rate": 4.945172324732496e-06,
+      "loss": 0.1427,
+      "step": 137
+    },
+    {
+      "epoch": 1.2560975609756098,
+      "grad_norm": 2.5540456771850586,
+      "learning_rate": 4.942180247926492e-06,
+      "loss": 0.143,
+      "step": 138
+    },
+    {
+      "epoch": 1.2652439024390243,
+      "grad_norm": 8.80340576171875,
+      "learning_rate": 4.939109637292563e-06,
+      "loss": 0.1569,
+      "step": 139
+    },
+    {
+      "epoch": 1.274390243902439,
+      "grad_norm": 3.1077523231506348,
+      "learning_rate": 4.935960591573136e-06,
+      "loss": 0.1313,
+      "step": 140
+    },
+    {
+      "epoch": 1.2835365853658536,
+      "grad_norm": 1.2506815195083618,
+      "learning_rate": 4.932733212032896e-06,
+      "loss": 0.137,
+      "step": 141
+    },
+    {
+      "epoch": 1.2926829268292683,
+      "grad_norm": 2.4397666454315186,
+      "learning_rate": 4.929427602455532e-06,
+      "loss": 0.1408,
+      "step": 142
+    },
+    {
+      "epoch": 1.3018292682926829,
+      "grad_norm": 1.7658663988113403,
+      "learning_rate": 4.926043869140392e-06,
+      "loss": 0.133,
+      "step": 143
+    },
+    {
+      "epoch": 1.3109756097560976,
+      "grad_norm": 1.0473108291625977,
+      "learning_rate": 4.922582120899072e-06,
+      "loss": 0.1358,
+      "step": 144
+    },
+    {
+      "epoch": 1.3201219512195121,
+      "grad_norm": 5.566607475280762,
+      "learning_rate": 4.919042469051916e-06,
+      "loss": 0.1237,
+      "step": 145
+    },
+    {
+      "epoch": 1.329268292682927,
+      "grad_norm": 1.0360139608383179,
+      "learning_rate": 4.915425027424429e-06,
+      "loss": 0.1274,
+      "step": 146
+    },
+    {
+      "epoch": 1.3384146341463414,
+      "grad_norm": 1.9091176986694336,
+      "learning_rate": 4.911729912343631e-06,
+      "loss": 0.1439,
+      "step": 147
+    },
+    {
+      "epoch": 1.3475609756097562,
+      "grad_norm": 1.5727638006210327,
+      "learning_rate": 4.907957242634299e-06,
+      "loss": 0.1259,
+      "step": 148
+    },
+    {
+      "epoch": 1.3567073170731707,
+      "grad_norm": 1.0574792623519897,
+      "learning_rate": 4.9041071396151585e-06,
+      "loss": 0.142,
+      "step": 149
+    },
+    {
+      "epoch": 1.3658536585365852,
+      "grad_norm": 8.020130157470703,
+      "learning_rate": 4.900179727094978e-06,
+      "loss": 0.1392,
+      "step": 150
+    },
+    {
+      "epoch": 1.375,
+      "grad_norm": 3.156255006790161,
+      "learning_rate": 4.896175131368587e-06,
+      "loss": 0.152,
+      "step": 151
+    },
+    {
+      "epoch": 1.3841463414634148,
+      "grad_norm": 2.3956990242004395,
+      "learning_rate": 4.892093481212817e-06,
+      "loss": 0.1244,
+      "step": 152
+    },
+    {
+      "epoch": 1.3932926829268293,
+      "grad_norm": 2.6058578491210938,
+      "learning_rate": 4.887934907882357e-06,
+      "loss": 0.1271,
+      "step": 153
+    },
+    {
+      "epoch": 1.4024390243902438,
+      "grad_norm": 1.7173289060592651,
+      "learning_rate": 4.883699545105535e-06,
+      "loss": 0.1257,
+      "step": 154
+    },
+    {
+      "epoch": 1.4115853658536586,
+      "grad_norm": 2.2237367630004883,
+      "learning_rate": 4.8793875290800186e-06,
+      "loss": 0.127,
+      "step": 155
+    },
+    {
+      "epoch": 1.420731707317073,
+      "grad_norm": 6.997623920440674,
+      "learning_rate": 4.874998998468433e-06,
+      "loss": 0.1454,
+      "step": 156
+    },
+    {
+      "epoch": 1.4298780487804879,
+      "grad_norm": 10.645139694213867,
+      "learning_rate": 4.870534094393903e-06,
+      "loss": 0.1402,
+      "step": 157
+    },
+    {
+      "epoch": 1.4390243902439024,
+      "grad_norm": 1.5808072090148926,
+      "learning_rate": 4.865992960435514e-06,
+      "loss": 0.1156,
+      "step": 158
+    },
+    {
+      "epoch": 1.4481707317073171,
+      "grad_norm": 0.990803062915802,
+      "learning_rate": 4.861375742623697e-06,
+      "loss": 0.1286,
+      "step": 159
+    },
+    {
+      "epoch": 1.4573170731707317,
+      "grad_norm": 1.5008662939071655,
+      "learning_rate": 4.856682589435531e-06,
+      "loss": 0.1282,
+      "step": 160
+    },
+    {
+      "epoch": 1.4664634146341464,
+      "grad_norm": 1.772304892539978,
+      "learning_rate": 4.851913651789968e-06,
+      "loss": 0.1308,
+      "step": 161
+    },
+    {
+      "epoch": 1.475609756097561,
+      "grad_norm": 1.201061487197876,
+      "learning_rate": 4.847069083042982e-06,
+      "loss": 0.1198,
+      "step": 162
+    },
+    {
+      "epoch": 1.4847560975609757,
+      "grad_norm": 0.6147828698158264,
+      "learning_rate": 4.8421490389826344e-06,
+      "loss": 0.1138,
+      "step": 163
+    },
+    {
+      "epoch": 1.4939024390243902,
+      "grad_norm": 1.3990256786346436,
+      "learning_rate": 4.837153677824068e-06,
+      "loss": 0.1205,
+      "step": 164
+    },
+    {
+      "epoch": 1.5030487804878048,
+      "grad_norm": 1.6977574825286865,
+      "learning_rate": 4.832083160204414e-06,
+      "loss": 0.1128,
+      "step": 165
+    },
+    {
+      "epoch": 1.5121951219512195,
+      "grad_norm": 2.4056453704833984,
+      "learning_rate": 4.826937649177632e-06,
+      "loss": 0.1184,
+      "step": 166
+    },
+    {
+      "epoch": 1.5213414634146343,
+      "grad_norm": 2.7455694675445557,
+      "learning_rate": 4.821717310209265e-06,
+      "loss": 0.123,
+      "step": 167
+    },
+    {
+      "epoch": 1.5304878048780488,
+      "grad_norm": 1.8243584632873535,
+      "learning_rate": 4.816422311171115e-06,
+      "loss": 0.1198,
+      "step": 168
+    },
+    {
+      "epoch": 1.5396341463414633,
+      "grad_norm": 0.8920577764511108,
+      "learning_rate": 4.811052822335849e-06,
+      "loss": 0.1088,
+      "step": 169
+    },
+    {
+      "epoch": 1.548780487804878,
+      "grad_norm": 1.6639117002487183,
+      "learning_rate": 4.805609016371522e-06,
+      "loss": 0.1231,
+      "step": 170
+    },
+    {
+      "epoch": 1.5579268292682928,
+      "grad_norm": 1.383026123046875,
+      "learning_rate": 4.800091068336024e-06,
+      "loss": 0.1207,
+      "step": 171
+    },
+    {
+      "epoch": 1.5670731707317072,
+      "grad_norm": 1.6599416732788086,
+      "learning_rate": 4.79449915567145e-06,
+      "loss": 0.1206,
+      "step": 172
+    },
+    {
+      "epoch": 1.576219512195122,
+      "grad_norm": 1.5868676900863647,
+      "learning_rate": 4.788833458198395e-06,
+      "loss": 0.1158,
+      "step": 173
+    },
+    {
+      "epoch": 1.5853658536585367,
+      "grad_norm": 1.0154211521148682,
+      "learning_rate": 4.783094158110175e-06,
+      "loss": 0.1091,
+      "step": 174
+    },
+    {
+      "epoch": 1.5945121951219512,
+      "grad_norm": 2.1514394283294678,
+      "learning_rate": 4.7772814399669585e-06,
+      "loss": 0.1109,
+      "step": 175
+    },
+    {
+      "epoch": 1.6036585365853657,
+      "grad_norm": 0.7475085258483887,
+      "learning_rate": 4.7713954906898415e-06,
+      "loss": 0.1162,
+      "step": 176
+    },
+    {
+      "epoch": 1.6128048780487805,
+      "grad_norm": 0.8523163795471191,
+      "learning_rate": 4.7654364995548325e-06,
+      "loss": 0.1238,
+      "step": 177
+    },
+    {
+      "epoch": 1.6219512195121952,
+      "grad_norm": 1.5456233024597168,
+      "learning_rate": 4.759404658186765e-06,
+      "loss": 0.1163,
+      "step": 178
+    },
+    {
+      "epoch": 1.6310975609756098,
+      "grad_norm": 1.123340129852295,
+      "learning_rate": 4.753300160553137e-06,
+      "loss": 0.1104,
+      "step": 179
+    },
+    {
+      "epoch": 1.6402439024390243,
+      "grad_norm": 1.16841721534729,
+      "learning_rate": 4.747123202957871e-06,
+      "loss": 0.1099,
+      "step": 180
+    },
+    {
+      "epoch": 1.649390243902439,
+      "grad_norm": 2.025221109390259,
+      "learning_rate": 4.740873984035009e-06,
+      "loss": 0.1157,
+      "step": 181
+    },
+    {
+      "epoch": 1.6585365853658538,
+      "grad_norm": 3.876469373703003,
+      "learning_rate": 4.734552704742314e-06,
+      "loss": 0.1122,
+      "step": 182
+    },
+    {
+      "epoch": 1.6676829268292683,
+      "grad_norm": 1.5562145709991455,
+      "learning_rate": 4.728159568354814e-06,
+      "loss": 0.1127,
+      "step": 183
+    },
+    {
+      "epoch": 1.6768292682926829,
+      "grad_norm": 1.4691755771636963,
+      "learning_rate": 4.721694780458266e-06,
+      "loss": 0.1182,
+      "step": 184
+    },
+    {
+      "epoch": 1.6859756097560976,
+      "grad_norm": 1.9529973268508911,
+      "learning_rate": 4.715158548942541e-06,
+      "loss": 0.1153,
+      "step": 185
+    },
+    {
+      "epoch": 1.6951219512195121,
+      "grad_norm": 1.9065771102905273,
+      "learning_rate": 4.708551083994944e-06,
+      "loss": 0.1163,
+      "step": 186
+    },
+    {
+      "epoch": 1.7042682926829267,
+      "grad_norm": 1.0964735746383667,
+      "learning_rate": 4.701872598093452e-06,
+      "loss": 0.114,
+      "step": 187
+    },
+    {
+      "epoch": 1.7134146341463414,
+      "grad_norm": 1.0885926485061646,
+      "learning_rate": 4.695123305999877e-06,
+      "loss": 0.114,
+      "step": 188
+    },
+    {
+      "epoch": 1.7225609756097562,
+      "grad_norm": 1.1777565479278564,
+      "learning_rate": 4.688303424752969e-06,
+      "loss": 0.1116,
+      "step": 189
+    },
+    {
+      "epoch": 1.7317073170731707,
+      "grad_norm": 1.3686976432800293,
+      "learning_rate": 4.6814131736614286e-06,
+      "loss": 0.1081,
+      "step": 190
+    },
+    {
+      "epoch": 1.7408536585365852,
+      "grad_norm": 0.8220391869544983,
+      "learning_rate": 4.674452774296861e-06,
+      "loss": 0.1091,
+      "step": 191
+    },
+    {
+      "epoch": 1.75,
+      "grad_norm": 1.073935866355896,
+      "learning_rate": 4.667422450486646e-06,
+      "loss": 0.1148,
+      "step": 192
+    },
+    {
+      "epoch": 1.7591463414634148,
+      "grad_norm": 1.389614224433899,
+      "learning_rate": 4.660322428306741e-06,
+      "loss": 0.1134,
+      "step": 193
+    },
+    {
+      "epoch": 1.7682926829268293,
+      "grad_norm": 1.349023699760437,
+      "learning_rate": 4.653152936074413e-06,
+      "loss": 0.1056,
+      "step": 194
+    },
+    {
+      "epoch": 1.7774390243902438,
+      "grad_norm": 1.0334906578063965,
+      "learning_rate": 4.645914204340898e-06,
+      "loss": 0.113,
+      "step": 195
+    },
+    {
+      "epoch": 1.7865853658536586,
+      "grad_norm": 18.6312313079834,
+      "learning_rate": 4.638606465883979e-06,
+      "loss": 0.1351,
+      "step": 196
+    },
+    {
+      "epoch": 1.7957317073170733,
+      "grad_norm": 0.7669345736503601,
+      "learning_rate": 4.6312299557005125e-06,
+      "loss": 0.1097,
+      "step": 197
+    },
+    {
+      "epoch": 1.8048780487804879,
+      "grad_norm": 2.1091127395629883,
+      "learning_rate": 4.623784910998862e-06,
+      "loss": 0.1122,
+      "step": 198
+    },
+    {
+      "epoch": 1.8140243902439024,
+      "grad_norm": 0.8031914830207825,
+      "learning_rate": 4.616271571191273e-06,
+      "loss": 0.1109,
+      "step": 199
+    },
+    {
+      "epoch": 1.8231707317073171,
+      "grad_norm": 2.1154661178588867,
+      "learning_rate": 4.608690177886177e-06,
+      "loss": 0.1131,
+      "step": 200
+    },
+    {
+      "epoch": 1.8323170731707317,
+      "grad_norm": 1.4946709871292114,
+      "learning_rate": 4.6010409748804165e-06,
+      "loss": 0.1136,
+      "step": 201
+    },
+    {
+      "epoch": 1.8414634146341462,
+      "grad_norm": 1.0509642362594604,
+      "learning_rate": 4.593324208151412e-06,
+      "loss": 0.1162,
+      "step": 202
+    },
+    {
+      "epoch": 1.850609756097561,
+      "grad_norm": 0.8742370009422302,
+      "learning_rate": 4.585540125849244e-06,
+      "loss": 0.1102,
+      "step": 203
+    },
+    {
+      "epoch": 1.8597560975609757,
+      "grad_norm": 1.1283056735992432,
+      "learning_rate": 4.577688978288681e-06,
+      "loss": 0.1104,
+      "step": 204
+    },
+    {
+      "epoch": 1.8689024390243902,
+      "grad_norm": 1.0081048011779785,
+      "learning_rate": 4.569771017941125e-06,
+      "loss": 0.1119,
+      "step": 205
+    },
+    {
+      "epoch": 1.8780487804878048,
+      "grad_norm": 0.7291656732559204,
+      "learning_rate": 4.561786499426494e-06,
+      "loss": 0.1103,
+      "step": 206
+    },
+    {
+      "epoch": 1.8871951219512195,
+      "grad_norm": 1.144096851348877,
+      "learning_rate": 4.5537356795050346e-06,
+      "loss": 0.1125,
+      "step": 207
+    },
+    {
+      "epoch": 1.8963414634146343,
+      "grad_norm": 0.7317456603050232,
+      "learning_rate": 4.545618817069066e-06,
+      "loss": 0.1105,
+      "step": 208
+    },
+    {
+      "epoch": 1.9054878048780488,
+      "grad_norm": 1.1636486053466797,
+      "learning_rate": 4.537436173134653e-06,
+      "loss": 0.1091,
+      "step": 209
+    },
+    {
+      "epoch": 1.9146341463414633,
+      "grad_norm": 0.6319037675857544,
+      "learning_rate": 4.529188010833212e-06,
+      "loss": 0.1088,
+      "step": 210
+    },
+    {
+      "epoch": 1.923780487804878,
+      "grad_norm": 0.7535589933395386,
+      "learning_rate": 4.520874595403053e-06,
+      "loss": 0.1041,
+      "step": 211
+    },
+    {
+      "epoch": 1.9329268292682928,
+      "grad_norm": 1.0461041927337646,
+      "learning_rate": 4.512496194180846e-06,
+      "loss": 0.1129,
+      "step": 212
+    },
+    {
+      "epoch": 1.9420731707317072,
+      "grad_norm": 0.6124874949455261,
+      "learning_rate": 4.5040530765930275e-06,
+      "loss": 0.1113,
+      "step": 213
+    },
+    {
+      "epoch": 1.951219512195122,
+      "grad_norm": 0.8516119122505188,
+      "learning_rate": 4.495545514147134e-06,
+      "loss": 0.1112,
+      "step": 214
+    },
+    {
+      "epoch": 1.9603658536585367,
+      "grad_norm": 0.8919075131416321,
+      "learning_rate": 4.486973780423073e-06,
+      "loss": 0.1089,
+      "step": 215
+    },
+    {
+      "epoch": 1.9695121951219512,
+      "grad_norm": 1.2183221578598022,
+      "learning_rate": 4.478338151064323e-06,
+      "loss": 0.1156,
+      "step": 216
+    },
+    {
+      "epoch": 1.9786585365853657,
+      "grad_norm": 1.0123646259307861,
+      "learning_rate": 4.4696389037690734e-06,
+      "loss": 0.1077,
+      "step": 217
+    },
+    {
+      "epoch": 1.9878048780487805,
+      "grad_norm": 0.8304896950721741,
+      "learning_rate": 4.460876318281291e-06,
+      "loss": 0.1051,
+      "step": 218
+    },
+    {
+      "epoch": 1.9969512195121952,
+      "grad_norm": 0.5879438519477844,
+      "learning_rate": 4.452050676381725e-06,
+      "loss": 0.1064,
+      "step": 219
+    },
+    {
+      "epoch": 2.0,
+      "grad_norm": 0.5879438519477844,
+      "learning_rate": 4.443162261878846e-06,
+      "loss": 0.0344,
+      "step": 220
+    },
+    {
+      "epoch": 2.0091463414634148,
+      "grad_norm": 0.8598087430000305,
+      "learning_rate": 4.434211360599721e-06,
+      "loss": 0.1071,
+      "step": 221
+    },
+    {
+      "epoch": 2.018292682926829,
+      "grad_norm": 0.5539699196815491,
+      "learning_rate": 4.425198260380818e-06,
+      "loss": 0.1095,
+      "step": 222
+    },
+    {
+      "epoch": 2.027439024390244,
+      "grad_norm": 0.4581831395626068,
+      "learning_rate": 4.416123251058756e-06,
+      "loss": 0.104,
+      "step": 223
+    },
+    {
+      "epoch": 2.0365853658536586,
+      "grad_norm": 0.550529420375824,
+      "learning_rate": 4.406986624460979e-06,
+      "loss": 0.1049,
+      "step": 224
+    },
+    {
+      "epoch": 2.0457317073170733,
+      "grad_norm": 0.749291181564331,
+      "learning_rate": 4.397788674396374e-06,
+      "loss": 0.1091,
+      "step": 225
+    },
+    {
+      "epoch": 2.0548780487804876,
+      "grad_norm": 0.7612536549568176,
+      "learning_rate": 4.3885296966458225e-06,
+      "loss": 0.1045,
+      "step": 226
+    },
+    {
+      "epoch": 2.0640243902439024,
+      "grad_norm": 0.6671517491340637,
+      "learning_rate": 4.37920998895269e-06,
+      "loss": 0.1057,
+      "step": 227
+    },
+    {
+      "epoch": 2.073170731707317,
+      "grad_norm": 0.5838347673416138,
+      "learning_rate": 4.369829851013251e-06,
+      "loss": 0.1061,
+      "step": 228
+    },
+    {
+      "epoch": 2.082317073170732,
+      "grad_norm": 1.937151312828064,
+      "learning_rate": 4.36038958446705e-06,
+      "loss": 0.1074,
+      "step": 229
+    },
+    {
+      "epoch": 2.091463414634146,
+      "grad_norm": 0.8083934187889099,
+      "learning_rate": 4.350889492887202e-06,
+      "loss": 0.1096,
+      "step": 230
+    },
+    {
+      "epoch": 2.100609756097561,
+      "grad_norm": 0.6810253262519836,
+      "learning_rate": 4.341329881770635e-06,
+      "loss": 0.1024,
+      "step": 231
+    },
+    {
+      "epoch": 2.1097560975609757,
+      "grad_norm": 0.8901084065437317,
+      "learning_rate": 4.331711058528259e-06,
+      "loss": 0.1064,
+      "step": 232
+    },
+    {
+      "epoch": 2.1189024390243905,
+      "grad_norm": 0.34419459104537964,
+      "learning_rate": 4.322033332475084e-06,
+      "loss": 0.1074,
+      "step": 233
+    },
+    {
+      "epoch": 2.1280487804878048,
+      "grad_norm": 0.8596321940422058,
+      "learning_rate": 4.312297014820275e-06,
+      "loss": 0.1061,
+      "step": 234
+    },
+    {
+      "epoch": 2.1371951219512195,
+      "grad_norm": 0.8391798138618469,
+      "learning_rate": 4.302502418657138e-06,
+      "loss": 0.107,
+      "step": 235
+    },
+    {
+      "epoch": 2.1463414634146343,
+      "grad_norm": 0.7193841338157654,
+      "learning_rate": 4.292649858953063e-06,
+      "loss": 0.1083,
+      "step": 236
+    },
+    {
+      "epoch": 2.1554878048780486,
+      "grad_norm": 1.5975165367126465,
+      "learning_rate": 4.282739652539384e-06,
+      "loss": 0.109,
+      "step": 237
+    },
+    {
+      "epoch": 2.1646341463414633,
+      "grad_norm": 0.9933533072471619,
+      "learning_rate": 4.272772118101195e-06,
+      "loss": 0.1073,
+      "step": 238
+    },
+    {
+      "epoch": 2.173780487804878,
+      "grad_norm": 0.9281901717185974,
+      "learning_rate": 4.262747576167106e-06,
+      "loss": 0.1066,
+      "step": 239
+    },
+    {
+      "epoch": 2.182926829268293,
+      "grad_norm": 3.2005884647369385,
+      "learning_rate": 4.2526663490989264e-06,
+      "loss": 0.1106,
+      "step": 240
+    },
+    {
+      "epoch": 2.192073170731707,
+      "grad_norm": 1.084905982017517,
+      "learning_rate": 4.24252876108131e-06,
+      "loss": 0.1047,
+      "step": 241
+    },
+    {
+      "epoch": 2.201219512195122,
+      "grad_norm": 0.5345124006271362,
+      "learning_rate": 4.232335138111321e-06,
+      "loss": 0.1027,
+      "step": 242
+    },
+    {
+      "epoch": 2.2103658536585367,
+      "grad_norm": 0.6235584020614624,
+      "learning_rate": 4.222085807987954e-06,
+      "loss": 0.1032,
+      "step": 243
+    },
+    {
+      "epoch": 2.2195121951219514,
+      "grad_norm": 0.9797384738922119,
+      "learning_rate": 4.211781100301596e-06,
+      "loss": 0.1082,
+      "step": 244
+    },
+    {
+      "epoch": 2.2286585365853657,
+      "grad_norm": 0.3665165901184082,
+      "learning_rate": 4.201421346423421e-06,
+      "loss": 0.1057,
+      "step": 245
+    },
+    {
+      "epoch": 2.2378048780487805,
+      "grad_norm": 1.232487440109253,
+      "learning_rate": 4.19100687949474e-06,
+      "loss": 0.1088,
+      "step": 246
+    },
+    {
+      "epoch": 2.2469512195121952,
+      "grad_norm": 0.8028481602668762,
+      "learning_rate": 4.180538034416287e-06,
+      "loss": 0.1119,
+      "step": 247
+    },
+    {
+      "epoch": 2.2560975609756095,
+      "grad_norm": 1.094025731086731,
+      "learning_rate": 4.1700151478374436e-06,
+      "loss": 0.1033,
+      "step": 248
+    },
+    {
+      "epoch": 2.2652439024390243,
+      "grad_norm": 1.0119608640670776,
+      "learning_rate": 4.159438558145425e-06,
+      "loss": 0.108,
+      "step": 249
+    },
+    {
+      "epoch": 2.274390243902439,
+      "grad_norm": 0.6864210963249207,
+      "learning_rate": 4.148808605454385e-06,
+      "loss": 0.1072,
+      "step": 250
+    },
+    {
+      "epoch": 2.283536585365854,
+      "grad_norm": 0.8415966629981995,
+      "learning_rate": 4.138125631594488e-06,
+      "loss": 0.1052,
+      "step": 251
+    },
+    {
+      "epoch": 2.292682926829268,
+      "grad_norm": 0.7150280475616455,
+      "learning_rate": 4.127389980100916e-06,
+      "loss": 0.1049,
+      "step": 252
+    },
+    {
+      "epoch": 2.301829268292683,
+      "grad_norm": 1.2521971464157104,
+      "learning_rate": 4.116601996202815e-06,
+      "loss": 0.1078,
+      "step": 253
+    },
+    {
+      "epoch": 2.3109756097560976,
+      "grad_norm": 0.8204450011253357,
+      "learning_rate": 4.1057620268122e-06,
+      "loss": 0.1078,
+      "step": 254
+    },
+    {
+      "epoch": 2.3201219512195124,
+      "grad_norm": 1.0992320775985718,
+      "learning_rate": 4.0948704205128e-06,
+      "loss": 0.1063,
+      "step": 255
+    },
+    {
+      "epoch": 2.3292682926829267,
+      "grad_norm": 0.4928065240383148,
+      "learning_rate": 4.08392752754884e-06,
+      "loss": 0.1076,
+      "step": 256
+    },
+    {
+      "epoch": 2.3384146341463414,
+      "grad_norm": 1.1588791608810425,
+      "learning_rate": 4.072933699813788e-06,
+      "loss": 0.1075,
+      "step": 257
+    },
+    {
+      "epoch": 2.347560975609756,
+      "grad_norm": 0.5810270309448242,
+      "learning_rate": 4.061889290839032e-06,
+      "loss": 0.107,
+      "step": 258
+    },
+    {
+      "epoch": 2.3567073170731705,
+      "grad_norm": 0.8428746461868286,
+      "learning_rate": 4.050794655782515e-06,
+      "loss": 0.1057,
+      "step": 259
+    },
+    {
+      "epoch": 2.3658536585365852,
+      "grad_norm": 1.4898464679718018,
+      "learning_rate": 4.039650151417316e-06,
+      "loss": 0.1043,
+      "step": 260
+    },
+    {
+      "epoch": 2.375,
+      "grad_norm": 0.8613987565040588,
+      "learning_rate": 4.028456136120169e-06,
+      "loss": 0.1011,
+      "step": 261
+    },
+    {
+      "epoch": 2.3841463414634148,
+      "grad_norm": 0.7771743535995483,
+      "learning_rate": 4.017212969859948e-06,
+      "loss": 0.1073,
+      "step": 262
+    },
+    {
+      "epoch": 2.3932926829268295,
+      "grad_norm": 0.4884893596172333,
+      "learning_rate": 4.0059210141860875e-06,
+      "loss": 0.104,
+      "step": 263
+    },
+    {
+      "epoch": 2.402439024390244,
+      "grad_norm": 0.8188875317573547,
+      "learning_rate": 3.994580632216952e-06,
+      "loss": 0.1043,
+      "step": 264
+    },
+    {
+      "epoch": 2.4115853658536586,
+      "grad_norm": 0.814239501953125,
+      "learning_rate": 3.9831921886281675e-06,
+      "loss": 0.1073,
+      "step": 265
+    },
+    {
+      "epoch": 2.4207317073170733,
+      "grad_norm": 1.029391884803772,
+      "learning_rate": 3.971756049640888e-06,
+      "loss": 0.103,
+      "step": 266
+    },
+    {
+      "epoch": 2.4298780487804876,
+      "grad_norm": 0.3826993405818939,
+      "learning_rate": 3.96027258301002e-06,
+      "loss": 0.1089,
+      "step": 267
+    },
+    {
+      "epoch": 2.4390243902439024,
+      "grad_norm": 0.5306946039199829,
+      "learning_rate": 3.9487421580123995e-06,
+      "loss": 0.1086,
+      "step": 268
+    },
+    {
+      "epoch": 2.448170731707317,
+      "grad_norm": 1.286549687385559,
+      "learning_rate": 3.937165145434914e-06,
+      "loss": 0.1085,
+      "step": 269
+    },
+    {
+      "epoch": 2.457317073170732,
+      "grad_norm": 1.5009950399398804,
+      "learning_rate": 3.92554191756258e-06,
+      "loss": 0.1068,
+      "step": 270
+    },
+    {
+      "epoch": 2.466463414634146,
+      "grad_norm": 0.9225317239761353,
+      "learning_rate": 3.9138728481665696e-06,
+      "loss": 0.1059,
+      "step": 271
+    },
+    {
+      "epoch": 2.475609756097561,
+      "grad_norm": 0.5062177181243896,
+      "learning_rate": 3.902158312492196e-06,
+      "loss": 0.1089,
+      "step": 272
+    },
+    {
+      "epoch": 2.4847560975609757,
+      "grad_norm": 0.6272971630096436,
+      "learning_rate": 3.8903986872468416e-06,
+      "loss": 0.1099,
+      "step": 273
+    },
+    {
+      "epoch": 2.4939024390243905,
+      "grad_norm": 1.207027554512024,
+      "learning_rate": 3.878594350587844e-06,
+      "loss": 0.1086,
+      "step": 274
+    },
+    {
+      "epoch": 2.5030487804878048,
+      "grad_norm": 0.8417999744415283,
+      "learning_rate": 3.866745682110343e-06,
+      "loss": 0.1091,
+      "step": 275
+    },
+    {
+      "epoch": 2.5121951219512195,
+      "grad_norm": 0.5926170945167542,
+      "learning_rate": 3.854853062835062e-06,
+      "loss": 0.1081,
+      "step": 276
+    },
+    {
+      "epoch": 2.5213414634146343,
+      "grad_norm": 0.5806312561035156,
+      "learning_rate": 3.842916875196066e-06,
+      "loss": 0.1057,
+      "step": 277
+    },
+    {
+      "epoch": 2.5304878048780486,
+      "grad_norm": 1.2453746795654297,
+      "learning_rate": 3.830937503028457e-06,
+      "loss": 0.1086,
+      "step": 278
+    },
+    {
+      "epoch": 2.5396341463414633,
+      "grad_norm": 0.32152408361434937,
+      "learning_rate": 3.818915331556032e-06,
+      "loss": 0.1076,
+      "step": 279
+    },
+    {
+      "epoch": 2.548780487804878,
+      "grad_norm": 0.5756719708442688,
+      "learning_rate": 3.8068507473789016e-06,
+      "loss": 0.1057,
+      "step": 280
+    },
+    {
+      "epoch": 2.557926829268293,
+      "grad_norm": 0.9345139861106873,
+      "learning_rate": 3.7947441384610473e-06,
+      "loss": 0.1043,
+      "step": 281
+    },
+    {
+      "epoch": 2.567073170731707,
+      "grad_norm": 0.6475419998168945,
+      "learning_rate": 3.7825958941178538e-06,
+      "loss": 0.1046,
+      "step": 282
+    },
+    {
+      "epoch": 2.576219512195122,
+      "grad_norm": 0.7345425486564636,
+      "learning_rate": 3.770406405003586e-06,
+      "loss": 0.1042,
+      "step": 283
+    },
+    {
+      "epoch": 2.5853658536585367,
+      "grad_norm": 0.769500195980072,
+      "learning_rate": 3.758176063098829e-06,
+      "loss": 0.1048,
+      "step": 284
+    },
+    {
+      "epoch": 2.5945121951219514,
+      "grad_norm": 0.5934444665908813,
+      "learning_rate": 3.745905261697881e-06,
+      "loss": 0.1086,
+      "step": 285
+    },
+    {
+      "epoch": 2.6036585365853657,
+      "grad_norm": 0.5718421936035156,
+      "learning_rate": 3.7335943953961063e-06,
+      "loss": 0.1027,
+      "step": 286
+    },
+    {
+      "epoch": 2.6128048780487805,
+      "grad_norm": 0.6114512085914612,
+      "learning_rate": 3.721243860077247e-06,
+      "loss": 0.1091,
+      "step": 287
+    },
+    {
+      "epoch": 2.6219512195121952,
+      "grad_norm": 0.5620512366294861,
+      "learning_rate": 3.708854052900693e-06,
+      "loss": 0.1088,
+      "step": 288
+    },
+    {
+      "epoch": 2.6310975609756095,
+      "grad_norm": 0.6152999997138977,
+      "learning_rate": 3.696425372288708e-06,
+      "loss": 0.1035,
+      "step": 289
+    },
+    {
+      "epoch": 2.6402439024390243,
+      "grad_norm": 0.9731057286262512,
+      "learning_rate": 3.68395821791362e-06,
+      "loss": 0.1066,
+      "step": 290
+    },
+    {
+      "epoch": 2.649390243902439,
+      "grad_norm": 0.49733713269233704,
+      "learning_rate": 3.6714529906849683e-06,
+      "loss": 0.1045,
+      "step": 291
+    },
+    {
+      "epoch": 2.658536585365854,
+      "grad_norm": 0.49706941843032837,
+      "learning_rate": 3.65891009273661e-06,
+      "loss": 0.1035,
+      "step": 292
+    },
+    {
+      "epoch": 2.6676829268292686,
+      "grad_norm": 0.32593467831611633,
+      "learning_rate": 3.6463299274137883e-06,
+      "loss": 0.1017,
+      "step": 293
+    },
+    {
+      "epoch": 2.676829268292683,
+      "grad_norm": 0.4569077491760254,
+      "learning_rate": 3.6337128992601665e-06,
+      "loss": 0.1061,
+      "step": 294
+    },
+    {
+      "epoch": 2.6859756097560976,
+      "grad_norm": 0.5540208220481873,
+      "learning_rate": 3.621059414004811e-06,
+      "loss": 0.106,
+      "step": 295
+    },
+    {
+      "epoch": 2.6951219512195124,
+      "grad_norm": 0.9319402575492859,
+      "learning_rate": 3.6083698785491537e-06,
+      "loss": 0.1042,
+      "step": 296
+    },
+    {
+      "epoch": 2.7042682926829267,
+      "grad_norm": 0.795711874961853,
+      "learning_rate": 3.595644700953898e-06,
+      "loss": 0.104,
+      "step": 297
+    },
+    {
+      "epoch": 2.7134146341463414,
+      "grad_norm": 0.4707932472229004,
+      "learning_rate": 3.582884290425903e-06,
+      "loss": 0.1087,
+      "step": 298
+    },
+    {
+      "epoch": 2.722560975609756,
+      "grad_norm": 0.7709797620773315,
+      "learning_rate": 3.5700890573050224e-06,
+      "loss": 0.107,
+      "step": 299
+    },
+    {
+      "epoch": 2.7317073170731705,
+      "grad_norm": 1.1342726945877075,
+      "learning_rate": 3.5572594130509074e-06,
+      "loss": 0.1029,
+      "step": 300
+    },
+    {
+      "epoch": 2.7408536585365852,
+      "grad_norm": 0.9055543541908264,
+      "learning_rate": 3.544395770229781e-06,
+      "loss": 0.1087,
+      "step": 301
+    },
+    {
+      "epoch": 2.75,
+      "grad_norm": 1.084656000137329,
+      "learning_rate": 3.531498542501161e-06,
+      "loss": 0.1044,
+      "step": 302
+    },
+    {
+      "epoch": 2.7591463414634148,
+      "grad_norm": 0.5991970300674438,
+      "learning_rate": 3.51856814460457e-06,
+      "loss": 0.1064,
+      "step": 303
+    },
+    {
+      "epoch": 2.7682926829268295,
+      "grad_norm": 1.7364325523376465,
+      "learning_rate": 3.5056049923461873e-06,
+      "loss": 0.1079,
+      "step": 304
+    },
+    {
+      "epoch": 2.777439024390244,
+      "grad_norm": 0.4037036895751953,
+      "learning_rate": 3.4926095025854863e-06,
+      "loss": 0.1058,
+      "step": 305
+    },
+    {
+      "epoch": 2.7865853658536586,
+      "grad_norm": 0.8944678902626038,
+      "learning_rate": 3.479582093221824e-06,
+      "loss": 0.1064,
+      "step": 306
+    },
+    {
+      "epoch": 2.7957317073170733,
+      "grad_norm": 0.5730510950088501,
+      "learning_rate": 3.4665231831810055e-06,
+      "loss": 0.1055,
+      "step": 307
+    },
+    {
+      "epoch": 2.8048780487804876,
+      "grad_norm": 0.519359827041626,
+      "learning_rate": 3.45343319240181e-06,
+      "loss": 0.1069,
+      "step": 308
+    },
+    {
+      "epoch": 2.8140243902439024,
+      "grad_norm": 0.6074658632278442,
+      "learning_rate": 3.44031254182249e-06,
+      "loss": 0.1058,
+      "step": 309
+    },
+    {
+      "epoch": 2.823170731707317,
+      "grad_norm": 0.37584108114242554,
+      "learning_rate": 3.4271616533672312e-06,
+      "loss": 0.1052,
+      "step": 310
+    },
+    {
+      "epoch": 2.8323170731707314,
+      "grad_norm": 0.7730855941772461,
+      "learning_rate": 3.413980949932589e-06,
+      "loss": 0.1037,
+      "step": 311
+    },
+    {
+      "epoch": 2.841463414634146,
+      "grad_norm": 0.9912458062171936,
+      "learning_rate": 3.4007708553738837e-06,
+      "loss": 0.1039,
+      "step": 312
+    },
+    {
+      "epoch": 2.850609756097561,
+      "grad_norm": 0.36861947178840637,
+      "learning_rate": 3.3875317944915765e-06,
+      "loss": 0.1083,
+      "step": 313
+    },
+    {
+      "epoch": 2.8597560975609757,
+      "grad_norm": 0.6069959402084351,
+      "learning_rate": 3.3742641930176044e-06,
+      "loss": 0.1063,
+      "step": 314
+    },
+    {
+      "epoch": 2.8689024390243905,
+      "grad_norm": 0.5478480458259583,
+      "learning_rate": 3.3609684776016932e-06,
+      "loss": 0.1033,
+      "step": 315
+    },
+    {
+      "epoch": 2.8780487804878048,
+      "grad_norm": 0.8352198004722595,
+      "learning_rate": 3.3476450757976353e-06,
+      "loss": 0.1026,
+      "step": 316
+    },
+    {
+      "epoch": 2.8871951219512195,
+      "grad_norm": 0.32641175389289856,
+      "learning_rate": 3.3342944160495405e-06,
+      "loss": 0.1065,
+      "step": 317
+    },
+    {
+      "epoch": 2.8963414634146343,
+      "grad_norm": 0.8076167702674866,
+      "learning_rate": 3.3209169276780612e-06,
+      "loss": 0.105,
+      "step": 318
+    },
+    {
+      "epoch": 2.9054878048780486,
+      "grad_norm": 0.3602111041545868,
+      "learning_rate": 3.307513040866584e-06,
+      "loss": 0.106,
+      "step": 319
+    },
+    {
+      "epoch": 2.9146341463414633,
+      "grad_norm": 0.227371945977211,
+      "learning_rate": 3.2940831866473967e-06,
+      "loss": 0.1029,
+      "step": 320
+    },
+    {
+      "epoch": 2.923780487804878,
+      "grad_norm": 0.5158879160881042,
+      "learning_rate": 3.280627796887828e-06,
+      "loss": 0.1046,
+      "step": 321
+    },
+    {
+      "epoch": 2.932926829268293,
+      "grad_norm": 1.0091784000396729,
+      "learning_rate": 3.2671473042763603e-06,
+      "loss": 0.1063,
+      "step": 322
+    },
+    {
+      "epoch": 2.942073170731707,
+      "grad_norm": 0.9310367107391357,
+      "learning_rate": 3.2536421423087137e-06,
+      "loss": 0.1015,
+      "step": 323
+    },
+    {
+      "epoch": 2.951219512195122,
+      "grad_norm": 0.7602329254150391,
+      "learning_rate": 3.2401127452739106e-06,
+      "loss": 0.1061,
+      "step": 324
+    },
+    {
+      "epoch": 2.9603658536585367,
+      "grad_norm": 0.3374696671962738,
+      "learning_rate": 3.2265595482403002e-06,
+      "loss": 0.1049,
+      "step": 325
+    },
+    {
+      "epoch": 2.9695121951219514,
+      "grad_norm": 0.8469128012657166,
+      "learning_rate": 3.212982987041582e-06,
+      "loss": 0.1059,
+      "step": 326
+    },
+    {
+      "epoch": 2.9786585365853657,
+      "grad_norm": 0.7546894550323486,
+      "learning_rate": 3.199383498262777e-06,
+      "loss": 0.1067,
+      "step": 327
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 654,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 6,
+  "save_steps": 109,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 9.884240649702605e+18,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}
diff --git a/checkpoint-327/training_args.bin b/checkpoint-327/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..24b9be216f1966bb6d081a79c30a21dced39cf99
--- /dev/null
+++ b/checkpoint-327/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:14990b8399bb8d39b746405b094f7eea1041d93488de8433b2d415de16ff957a
+size 7928
diff --git a/checkpoint-327/zero_to_fp32.py b/checkpoint-327/zero_to_fp32.py
new file mode 100644
index 0000000000000000000000000000000000000000..24cc342e78d1a006c782b3a4cd68d9ce786d8fd8
--- /dev/null
+++ b/checkpoint-327/zero_to_fp32.py
@@ -0,0 +1,604 @@
+#!/usr/bin/env python
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
+# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
+# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
+# application.
+#
+# example: python zero_to_fp32.py . pytorch_model.bin
+
+import argparse
+import torch
+import glob
+import math
+import os
+import re
+from collections import OrderedDict
+from dataclasses import dataclass
+
+# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
+# DeepSpeed data structures it has to be available in the current python environment.
+from deepspeed.utils import logger
+from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
+                                            FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
+                                            FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS)
+
+
+@dataclass
+class zero_model_state:
+    buffers: dict()
+    param_shapes: dict()
+    shared_params: list
+    ds_version: int
+    frozen_param_shapes: dict()
+    frozen_param_fragments: dict()
+
+
+debug = 0
+
+# load to cpu
+device = torch.device('cpu')
+
+
+def atoi(text):
+    return int(text) if text.isdigit() else text
+
+
+def natural_keys(text):
+    '''
+    alist.sort(key=natural_keys) sorts in human order
+    http://nedbatchelder.com/blog/200712/human_sorting.html
+    (See Toothy's implementation in the comments)
+    '''
+    return [atoi(c) for c in re.split(r'(\d+)', text)]
+
+
+def get_model_state_file(checkpoint_dir, zero_stage):
+    if not os.path.isdir(checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
+
+    # there should be only one file
+    if zero_stage <= 2:
+        file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
+    elif zero_stage == 3:
+        file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
+
+    if not os.path.exists(file):
+        raise FileNotFoundError(f"can't find model states file at '{file}'")
+
+    return file
+
+
+def get_checkpoint_files(checkpoint_dir, glob_pattern):
+    # XXX: need to test that this simple glob rule works for multi-node setup too
+    ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
+
+    if len(ckpt_files) == 0:
+        raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
+
+    return ckpt_files
+
+
+def get_optim_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
+
+
+def get_model_state_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
+
+
+def parse_model_states(files):
+    zero_model_states = []
+    for file in files:
+        state_dict = torch.load(file, map_location=device)
+
+        if BUFFER_NAMES not in state_dict:
+            raise ValueError(f"{file} is not a model state checkpoint")
+        buffer_names = state_dict[BUFFER_NAMES]
+        if debug:
+            print("Found buffers:", buffer_names)
+
+        # recover just the buffers while restoring them to fp32 if they were saved in fp16
+        buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
+        param_shapes = state_dict[PARAM_SHAPES]
+
+        # collect parameters that are included in param_shapes
+        param_names = []
+        for s in param_shapes:
+            for name in s.keys():
+                param_names.append(name)
+
+        # update with frozen parameters
+        frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
+        if frozen_param_shapes is not None:
+            if debug:
+                print(f"Found frozen_param_shapes: {frozen_param_shapes}")
+            param_names += list(frozen_param_shapes.keys())
+
+        # handle shared params
+        shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
+
+        ds_version = state_dict.get(DS_VERSION, None)
+
+        frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
+
+        z_model_state = zero_model_state(buffers=buffers,
+                                         param_shapes=param_shapes,
+                                         shared_params=shared_params,
+                                         ds_version=ds_version,
+                                         frozen_param_shapes=frozen_param_shapes,
+                                         frozen_param_fragments=frozen_param_fragments)
+        zero_model_states.append(z_model_state)
+
+    return zero_model_states
+
+
+def parse_optim_states(files, ds_checkpoint_dir):
+
+    total_files = len(files)
+    state_dicts = []
+    for f in files:
+        state_dict = torch.load(f, map_location=device)
+        # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+        # and also handle the case where it was already removed by another helper script
+        state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+        state_dicts.append(state_dict)
+
+    if not ZERO_STAGE in state_dicts[0][OPTIMIZER_STATE_DICT]:
+        raise ValueError(f"{files[0]} is not a zero checkpoint")
+    zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
+    world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
+
+    # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
+    # parameters can be different from data parallelism for non-expert parameters. So we can just
+    # use the max of the partition_count to get the dp world_size.
+
+    if type(world_size) is list:
+        world_size = max(world_size)
+
+    if world_size != total_files:
+        raise ValueError(
+            f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
+            "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
+        )
+
+    # the groups are named differently in each stage
+    if zero_stage <= 2:
+        fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
+    elif zero_stage == 3:
+        fp32_groups_key = FP32_FLAT_GROUPS
+    else:
+        raise ValueError(f"unknown zero stage {zero_stage}")
+
+    if zero_stage <= 2:
+        fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
+    elif zero_stage == 3:
+        # if there is more than one param group, there will be multiple flattened tensors - one
+        # flattened tensor per group - for simplicity merge them into a single tensor
+        #
+        # XXX: could make the script more memory efficient for when there are multiple groups - it
+        # will require matching the sub-lists of param_shapes for each param group flattened tensor
+
+        fp32_flat_groups = [
+            torch.cat(state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key], 0) for i in range(len(state_dicts))
+        ]
+
+    return zero_stage, world_size, fp32_flat_groups
+
+
+def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
+    """
+    Returns fp32 state_dict reconstructed from ds checkpoint
+
+    Args:
+        - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
+
+    """
+    print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
+
+    optim_files = get_optim_files(ds_checkpoint_dir)
+    zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
+    print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
+
+    model_files = get_model_state_files(ds_checkpoint_dir)
+
+    zero_model_states = parse_model_states(model_files)
+    print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
+
+    if zero_stage <= 2:
+        return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+    elif zero_stage == 3:
+        return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+
+
+def _zero2_merge_frozen_params(state_dict, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+    frozen_param_fragments = zero_model_states[0].frozen_param_fragments
+
+    if debug:
+        num_elem = sum(s.numel() for s in frozen_param_shapes.values())
+        print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        state_dict[name] = frozen_param_fragments[name]
+
+        if debug:
+            print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _has_callable(obj, fn):
+    attr = getattr(obj, fn, None)
+    return callable(attr)
+
+
+def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+
+    # Reconstruction protocol:
+    #
+    # XXX: document this
+
+    if debug:
+        for i in range(world_size):
+            for j in range(len(fp32_flat_groups[0])):
+                print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
+
+    # XXX: memory usage doubles here (zero2)
+    num_param_groups = len(fp32_flat_groups[0])
+    merged_single_partition_of_fp32_groups = []
+    for i in range(num_param_groups):
+        merged_partitions = [sd[i] for sd in fp32_flat_groups]
+        full_single_fp32_vector = torch.cat(merged_partitions, 0)
+        merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
+    avail_numel = sum(
+        [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
+
+    if debug:
+        wanted_params = sum([len(shapes) for shapes in param_shapes])
+        wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
+        # not asserting if there is a mismatch due to possible padding
+        print(f"Have {avail_numel} numels to process.")
+        print(f"Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    total_numel = 0
+    total_params = 0
+    for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
+        offset = 0
+        avail_numel = full_single_fp32_vector.numel()
+        for name, shape in shapes.items():
+
+            unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
+            total_numel += unpartitioned_numel
+            total_params += 1
+
+            if debug:
+                print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+            state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
+            offset += unpartitioned_numel
+
+        # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
+        # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
+        # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
+        # live optimizer object, so we are checking that the numbers are within the right range
+        align_to = 2 * world_size
+
+        def zero2_align(x):
+            return align_to * math.ceil(x / align_to)
+
+        if debug:
+            print(f"original offset={offset}, avail_numel={avail_numel}")
+
+        offset = zero2_align(offset)
+        avail_numel = zero2_align(avail_numel)
+
+        if debug:
+            print(f"aligned  offset={offset}, avail_numel={avail_numel}")
+
+        # Sanity check
+        if offset != avail_numel:
+            raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero2_merge_frozen_params(state_dict, zero_model_states)
+
+    _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def zero3_partitioned_param_info(unpartitioned_numel, world_size):
+    remainder = unpartitioned_numel % world_size
+    padding_numel = (world_size - remainder) if remainder else 0
+    partitioned_numel = math.ceil(unpartitioned_numel / world_size)
+    return partitioned_numel, padding_numel
+
+
+def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    if debug:
+        for i in range(world_size):
+            num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
+            print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in zero_model_states[0].frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
+        state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
+
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+    avail_numel = fp32_flat_groups[0].numel() * world_size
+    # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
+    # param, re-consolidating each param, while dealing with padding if any
+
+    # merge list of dicts, preserving order
+    param_shapes = {k: v for d in param_shapes for k, v in d.items()}
+
+    if debug:
+        for i in range(world_size):
+            print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
+
+        wanted_params = len(param_shapes)
+        wanted_numel = sum(shape.numel() for shape in param_shapes.values())
+        # not asserting if there is a mismatch due to possible padding
+        avail_numel = fp32_flat_groups[0].numel() * world_size
+        print(f"Trainable params: Have {avail_numel} numels to process.")
+        print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    offset = 0
+    total_numel = 0
+    total_params = 0
+    for name, shape in param_shapes.items():
+
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+        total_params += 1
+
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+        # XXX: memory usage doubles here
+        state_dict[name] = torch.cat(
+            tuple(fp32_flat_groups[i].narrow(0, offset, partitioned_numel) for i in range(world_size)),
+            0).narrow(0, 0, unpartitioned_numel).view(shape)
+        offset += partitioned_numel
+
+    offset *= world_size
+
+    # Sanity check
+    if offset != avail_numel:
+        raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
+
+    _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag=None, exclude_frozen_parameters=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
+    ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
+    via a model hub.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+
+    Returns:
+        - pytorch ``state_dict``
+
+    Note: this approach may not work if your application doesn't have sufficient free CPU memory and
+    you may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
+    the checkpoint.
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+        # do the training and checkpoint saving
+        state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
+        model = model.cpu() # move to cpu
+        model.load_state_dict(state_dict)
+        # submit to model hub or save the model to share with others
+
+    In this example the ``model`` will no longer be usable in the deepspeed context of the same
+    application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
+
+    """
+    if tag is None:
+        latest_path = os.path.join(checkpoint_dir, 'latest')
+        if os.path.isfile(latest_path):
+            with open(latest_path, 'r') as fd:
+                tag = fd.read().strip()
+        else:
+            raise ValueError(f"Unable to find 'latest' file at {latest_path}")
+
+    ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
+
+    if not os.path.isdir(ds_checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
+
+    return _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
+
+
+def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, output_file, tag=None, exclude_frozen_parameters=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
+    loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``output_file``: path to the pytorch fp32 state_dict output file (e.g. path/pytorch_model.bin)
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+    """
+
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag, exclude_frozen_parameters)
+    print(f"Saving fp32 state dict to {output_file}")
+    torch.save(state_dict, output_file)
+
+
+def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
+    """
+    1. Put the provided model to cpu
+    2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
+    3. Load it into the provided model
+
+    Args:
+        - ``model``: the model object to update
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+
+    Returns:
+        - ``model`: modified model
+
+    Make sure you have plenty of CPU memory available before you call this function. If you don't
+    have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
+    conveniently placed for you in the checkpoint folder.
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
+        model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
+        # submit to model hub or save the model to share with others
+
+    Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
+    of the same application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    """
+    logger.info(f"Extracting fp32 weights")
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+
+    logger.info(f"Overwriting model with fp32 weights")
+    model = model.cpu()
+    model.load_state_dict(state_dict, strict=False)
+
+    return model
+
+
+if __name__ == "__main__":
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("checkpoint_dir",
+                        type=str,
+                        help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+    parser.add_argument(
+        "output_file",
+        type=str,
+        help="path to the pytorch fp32 state_dict output file (e.g. path/checkpoint-12/pytorch_model.bin)")
+    parser.add_argument("-t",
+                        "--tag",
+                        type=str,
+                        default=None,
+                        help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
+    parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+    parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
+    args = parser.parse_args()
+
+    debug = args.debug
+
+    convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
+                                               args.output_file,
+                                               tag=args.tag,
+                                               exclude_frozen_parameters=args.exclude_frozen_parameters)
diff --git a/checkpoint-436/README.md b/checkpoint-436/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..f4a3934800eeb082a0cb833d7b6af4f68eed3615
--- /dev/null
+++ b/checkpoint-436/README.md
@@ -0,0 +1,202 @@
+---
+base_model: nvidia/Llama-3_3-Nemotron-Super-49B-v1
+library_name: peft
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.15.0
\ No newline at end of file
diff --git a/checkpoint-436/adapter_config.json b/checkpoint-436/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..d7d7c196589a89c5b284ba682f27212262154f44
--- /dev/null
+++ b/checkpoint-436/adapter_config.json
@@ -0,0 +1,42 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "nvidia/Llama-3_3-Nemotron-Super-49B-v1",
+  "bias": "none",
+  "corda_config": null,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": null,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 512,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": [
+    "embed_tokens",
+    "lm_head"
+  ],
+  "peft_type": "LORA",
+  "r": 256,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "o_proj",
+    "gate_proj",
+    "q_proj",
+    "k_proj",
+    "v_proj",
+    "up_proj",
+    "down_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_rslora": false
+}
\ No newline at end of file
diff --git a/checkpoint-436/adapter_model.safetensors b/checkpoint-436/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..203c48b20e7884dc002db6a04ea20064129aa5bc
--- /dev/null
+++ b/checkpoint-436/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a375fa275fdb0c0a38f1a17ff38fe72fcccbaea53bfd6ea77d30bb0961a4d832
+size 9016826528
diff --git a/checkpoint-436/global_step434/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/checkpoint-436/global_step434/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..3e28fe8a994b230063a10c0af8230aef29053d8b
--- /dev/null
+++ b/checkpoint-436/global_step434/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:829cd54972a20f22abb36949bb0f2f2ec09c604dc9512c2a0ef8c13ae19aba01
+size 27050164444
diff --git a/checkpoint-436/global_step434/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/checkpoint-436/global_step434/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ebf27b4ccd1b1a613c797da15f186c8e6b60c4dd
--- /dev/null
+++ b/checkpoint-436/global_step434/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4cd7494f8022d562f063364353349d2cff7826b56c5cca57700940c5e4852f91
+size 27050169884
diff --git a/checkpoint-436/global_step434/mp_rank_00_model_states.pt b/checkpoint-436/global_step434/mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..9af07c2a18001aa20e7cd3651842146df4bbd472
--- /dev/null
+++ b/checkpoint-436/global_step434/mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3128d21e35f1e7a1aeeb1f14b7f179da6f11875e663bc27e6fa37c08150fe154
+size 9776788601
diff --git a/checkpoint-436/latest b/checkpoint-436/latest
new file mode 100644
index 0000000000000000000000000000000000000000..c3f3cfcb7ac734fd4ac1ad448f357785e3c96be2
--- /dev/null
+++ b/checkpoint-436/latest
@@ -0,0 +1 @@
+global_step434
\ No newline at end of file
diff --git a/checkpoint-436/rng_state_0.pth b/checkpoint-436/rng_state_0.pth
new file mode 100644
index 0000000000000000000000000000000000000000..799581ffc284eed44fc1e08d6e02e62676204987
--- /dev/null
+++ b/checkpoint-436/rng_state_0.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:53b43b86434d5ec2607b574a58dca4d4dffd1f385e51e5cc9446125f1178f571
+size 14512
diff --git a/checkpoint-436/rng_state_1.pth b/checkpoint-436/rng_state_1.pth
new file mode 100644
index 0000000000000000000000000000000000000000..658bda799ff09cd4202cecbadae4533207e4864d
--- /dev/null
+++ b/checkpoint-436/rng_state_1.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d481da70b20f261559f8905748fba496b3e5ac3532bb4365ddc3f86459c63cb6
+size 14512
diff --git a/checkpoint-436/scheduler.pt b/checkpoint-436/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..86d013ce67fc5f62a19b6b34a9d0f842d51a5a7d
--- /dev/null
+++ b/checkpoint-436/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ff9e8679f413de4253d1c16f0e6755d63f602d4b18769ceed942f8204f2121d2
+size 1064
diff --git a/checkpoint-436/special_tokens_map.json b/checkpoint-436/special_tokens_map.json
new file mode 100644
index 0000000000000000000000000000000000000000..278b7f0f84be865c4687700ee7b3c63d89a51e18
--- /dev/null
+++ b/checkpoint-436/special_tokens_map.json
@@ -0,0 +1,23 @@
+{
+  "bos_token": {
+    "content": "<|begin_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|eot_id|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|end_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}
diff --git a/checkpoint-436/tokenizer.json b/checkpoint-436/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..1c1d8d5c9024994f1d3b00f9662b8dd89ca13cf2
--- /dev/null
+++ b/checkpoint-436/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920
diff --git a/checkpoint-436/tokenizer_config.json b/checkpoint-436/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..edd01b980c1db496ea102a51c972ee8f5d1a2c74
--- /dev/null
+++ b/checkpoint-436/tokenizer_config.json
@@ -0,0 +1,2064 @@
+{
+  "added_tokens_decoder": {
+    "128000": {
+      "content": "<|begin_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128001": {
+      "content": "<|end_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128002": {
+      "content": "<|reserved_special_token_0|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128003": {
+      "content": "<|reserved_special_token_1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128004": {
+      "content": "<|finetune_right_pad_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128005": {
+      "content": "<|reserved_special_token_2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128006": {
+      "content": "<|start_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128007": {
+      "content": "<|end_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128008": {
+      "content": "<|eom_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128009": {
+      "content": "<|eot_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128010": {
+      "content": "<|python_tag|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128011": {
+      "content": "<|reserved_special_token_3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128012": {
+      "content": "<|reserved_special_token_4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128013": {
+      "content": "<|reserved_special_token_5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128014": {
+      "content": "<|reserved_special_token_6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128015": {
+      "content": "<|reserved_special_token_7|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128016": {
+      "content": "<|reserved_special_token_8|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128017": {
+      "content": "<|reserved_special_token_9|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128018": {
+      "content": "<|reserved_special_token_10|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128019": {
+      "content": "<|reserved_special_token_11|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128020": {
+      "content": "<|reserved_special_token_12|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128021": {
+      "content": "<|reserved_special_token_13|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128022": {
+      "content": "<|reserved_special_token_14|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128023": {
+      "content": "<|reserved_special_token_15|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128024": {
+      "content": "<|reserved_special_token_16|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128025": {
+      "content": "<|reserved_special_token_17|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128026": {
+      "content": "<|reserved_special_token_18|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128027": {
+      "content": "<|reserved_special_token_19|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128028": {
+      "content": "<|reserved_special_token_20|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128029": {
+      "content": "<|reserved_special_token_21|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128030": {
+      "content": "<|reserved_special_token_22|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128031": {
+      "content": "<|reserved_special_token_23|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128032": {
+      "content": "<|reserved_special_token_24|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128033": {
+      "content": "<|reserved_special_token_25|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128034": {
+      "content": "<|reserved_special_token_26|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128035": {
+      "content": "<|reserved_special_token_27|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128036": {
+      "content": "<|reserved_special_token_28|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128037": {
+      "content": "<|reserved_special_token_29|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128038": {
+      "content": "<|reserved_special_token_30|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128039": {
+      "content": "<|reserved_special_token_31|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128040": {
+      "content": "<|reserved_special_token_32|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128041": {
+      "content": "<|reserved_special_token_33|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128042": {
+      "content": "<|reserved_special_token_34|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128043": {
+      "content": "<|reserved_special_token_35|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128044": {
+      "content": "<|reserved_special_token_36|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128045": {
+      "content": "<|reserved_special_token_37|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128046": {
+      "content": "<|reserved_special_token_38|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128047": {
+      "content": "<|reserved_special_token_39|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128048": {
+      "content": "<|reserved_special_token_40|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128049": {
+      "content": "<|reserved_special_token_41|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128050": {
+      "content": "<|reserved_special_token_42|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128051": {
+      "content": "<|reserved_special_token_43|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128052": {
+      "content": "<|reserved_special_token_44|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128053": {
+      "content": "<|reserved_special_token_45|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128054": {
+      "content": "<|reserved_special_token_46|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128055": {
+      "content": "<|reserved_special_token_47|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128056": {
+      "content": "<|reserved_special_token_48|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128057": {
+      "content": "<|reserved_special_token_49|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128058": {
+      "content": "<|reserved_special_token_50|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128059": {
+      "content": "<|reserved_special_token_51|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128060": {
+      "content": "<|reserved_special_token_52|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128061": {
+      "content": "<|reserved_special_token_53|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128062": {
+      "content": "<|reserved_special_token_54|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128063": {
+      "content": "<|reserved_special_token_55|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128064": {
+      "content": "<|reserved_special_token_56|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128065": {
+      "content": "<|reserved_special_token_57|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128066": {
+      "content": "<|reserved_special_token_58|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128067": {
+      "content": "<|reserved_special_token_59|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128068": {
+      "content": "<|reserved_special_token_60|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128069": {
+      "content": "<|reserved_special_token_61|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128070": {
+      "content": "<|reserved_special_token_62|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128071": {
+      "content": "<|reserved_special_token_63|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128072": {
+      "content": "<|reserved_special_token_64|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128073": {
+      "content": "<|reserved_special_token_65|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128074": {
+      "content": "<|reserved_special_token_66|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128075": {
+      "content": "<|reserved_special_token_67|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128076": {
+      "content": "<|reserved_special_token_68|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128077": {
+      "content": "<|reserved_special_token_69|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128078": {
+      "content": "<|reserved_special_token_70|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128079": {
+      "content": "<|reserved_special_token_71|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128080": {
+      "content": "<|reserved_special_token_72|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128081": {
+      "content": "<|reserved_special_token_73|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128082": {
+      "content": "<|reserved_special_token_74|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128083": {
+      "content": "<|reserved_special_token_75|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128084": {
+      "content": "<|reserved_special_token_76|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128085": {
+      "content": "<|reserved_special_token_77|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128086": {
+      "content": "<|reserved_special_token_78|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128087": {
+      "content": "<|reserved_special_token_79|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128088": {
+      "content": "<|reserved_special_token_80|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128089": {
+      "content": "<|reserved_special_token_81|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128090": {
+      "content": "<|reserved_special_token_82|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128091": {
+      "content": "<|reserved_special_token_83|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128092": {
+      "content": "<|reserved_special_token_84|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128093": {
+      "content": "<|reserved_special_token_85|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128094": {
+      "content": "<|reserved_special_token_86|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128095": {
+      "content": "<|reserved_special_token_87|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128096": {
+      "content": "<|reserved_special_token_88|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128097": {
+      "content": "<|reserved_special_token_89|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128098": {
+      "content": "<|reserved_special_token_90|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128099": {
+      "content": "<|reserved_special_token_91|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128100": {
+      "content": "<|reserved_special_token_92|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128101": {
+      "content": "<|reserved_special_token_93|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128102": {
+      "content": "<|reserved_special_token_94|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128103": {
+      "content": "<|reserved_special_token_95|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128104": {
+      "content": "<|reserved_special_token_96|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128105": {
+      "content": "<|reserved_special_token_97|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128106": {
+      "content": "<|reserved_special_token_98|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128107": {
+      "content": "<|reserved_special_token_99|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128108": {
+      "content": "<|reserved_special_token_100|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128109": {
+      "content": "<|reserved_special_token_101|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128110": {
+      "content": "<|reserved_special_token_102|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128111": {
+      "content": "<|reserved_special_token_103|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128112": {
+      "content": "<|reserved_special_token_104|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128113": {
+      "content": "<|reserved_special_token_105|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128114": {
+      "content": "<|reserved_special_token_106|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128115": {
+      "content": "<|reserved_special_token_107|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128116": {
+      "content": "<|reserved_special_token_108|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128117": {
+      "content": "<|reserved_special_token_109|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128118": {
+      "content": "<|reserved_special_token_110|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128119": {
+      "content": "<|reserved_special_token_111|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128120": {
+      "content": "<|reserved_special_token_112|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128121": {
+      "content": "<|reserved_special_token_113|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128122": {
+      "content": "<|reserved_special_token_114|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128123": {
+      "content": "<|reserved_special_token_115|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128124": {
+      "content": "<|reserved_special_token_116|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128125": {
+      "content": "<|reserved_special_token_117|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128126": {
+      "content": "<|reserved_special_token_118|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128127": {
+      "content": "<|reserved_special_token_119|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128128": {
+      "content": "<|reserved_special_token_120|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128129": {
+      "content": "<|reserved_special_token_121|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128130": {
+      "content": "<|reserved_special_token_122|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128131": {
+      "content": "<|reserved_special_token_123|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128132": {
+      "content": "<|reserved_special_token_124|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128133": {
+      "content": "<|reserved_special_token_125|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128134": {
+      "content": "<|reserved_special_token_126|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128135": {
+      "content": "<|reserved_special_token_127|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128136": {
+      "content": "<|reserved_special_token_128|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128137": {
+      "content": "<|reserved_special_token_129|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128138": {
+      "content": "<|reserved_special_token_130|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128139": {
+      "content": "<|reserved_special_token_131|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128140": {
+      "content": "<|reserved_special_token_132|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128141": {
+      "content": "<|reserved_special_token_133|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128142": {
+      "content": "<|reserved_special_token_134|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128143": {
+      "content": "<|reserved_special_token_135|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128144": {
+      "content": "<|reserved_special_token_136|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128145": {
+      "content": "<|reserved_special_token_137|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128146": {
+      "content": "<|reserved_special_token_138|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128147": {
+      "content": "<|reserved_special_token_139|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128148": {
+      "content": "<|reserved_special_token_140|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128149": {
+      "content": "<|reserved_special_token_141|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128150": {
+      "content": "<|reserved_special_token_142|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128151": {
+      "content": "<|reserved_special_token_143|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128152": {
+      "content": "<|reserved_special_token_144|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128153": {
+      "content": "<|reserved_special_token_145|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128154": {
+      "content": "<|reserved_special_token_146|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128155": {
+      "content": "<|reserved_special_token_147|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128156": {
+      "content": "<|reserved_special_token_148|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128157": {
+      "content": "<|reserved_special_token_149|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128158": {
+      "content": "<|reserved_special_token_150|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128159": {
+      "content": "<|reserved_special_token_151|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128160": {
+      "content": "<|reserved_special_token_152|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128161": {
+      "content": "<|reserved_special_token_153|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128162": {
+      "content": "<|reserved_special_token_154|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128163": {
+      "content": "<|reserved_special_token_155|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128164": {
+      "content": "<|reserved_special_token_156|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128165": {
+      "content": "<|reserved_special_token_157|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128166": {
+      "content": "<|reserved_special_token_158|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128167": {
+      "content": "<|reserved_special_token_159|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128168": {
+      "content": "<|reserved_special_token_160|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128169": {
+      "content": "<|reserved_special_token_161|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128170": {
+      "content": "<|reserved_special_token_162|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128171": {
+      "content": "<|reserved_special_token_163|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128172": {
+      "content": "<|reserved_special_token_164|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128173": {
+      "content": "<|reserved_special_token_165|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128174": {
+      "content": "<|reserved_special_token_166|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128175": {
+      "content": "<|reserved_special_token_167|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128176": {
+      "content": "<|reserved_special_token_168|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128177": {
+      "content": "<|reserved_special_token_169|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128178": {
+      "content": "<|reserved_special_token_170|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128179": {
+      "content": "<|reserved_special_token_171|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128180": {
+      "content": "<|reserved_special_token_172|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128181": {
+      "content": "<|reserved_special_token_173|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128182": {
+      "content": "<|reserved_special_token_174|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128183": {
+      "content": "<|reserved_special_token_175|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128184": {
+      "content": "<|reserved_special_token_176|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128185": {
+      "content": "<|reserved_special_token_177|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128186": {
+      "content": "<|reserved_special_token_178|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128187": {
+      "content": "<|reserved_special_token_179|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128188": {
+      "content": "<|reserved_special_token_180|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128189": {
+      "content": "<|reserved_special_token_181|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128190": {
+      "content": "<|reserved_special_token_182|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128191": {
+      "content": "<|reserved_special_token_183|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128192": {
+      "content": "<|reserved_special_token_184|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128193": {
+      "content": "<|reserved_special_token_185|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128194": {
+      "content": "<|reserved_special_token_186|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128195": {
+      "content": "<|reserved_special_token_187|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128196": {
+      "content": "<|reserved_special_token_188|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128197": {
+      "content": "<|reserved_special_token_189|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128198": {
+      "content": "<|reserved_special_token_190|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128199": {
+      "content": "<|reserved_special_token_191|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128200": {
+      "content": "<|reserved_special_token_192|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128201": {
+      "content": "<|reserved_special_token_193|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128202": {
+      "content": "<|reserved_special_token_194|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128203": {
+      "content": "<|reserved_special_token_195|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128204": {
+      "content": "<|reserved_special_token_196|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128205": {
+      "content": "<|reserved_special_token_197|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128206": {
+      "content": "<|reserved_special_token_198|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128207": {
+      "content": "<|reserved_special_token_199|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128208": {
+      "content": "<|reserved_special_token_200|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128209": {
+      "content": "<|reserved_special_token_201|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128210": {
+      "content": "<|reserved_special_token_202|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128211": {
+      "content": "<|reserved_special_token_203|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128212": {
+      "content": "<|reserved_special_token_204|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128213": {
+      "content": "<|reserved_special_token_205|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128214": {
+      "content": "<|reserved_special_token_206|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128215": {
+      "content": "<|reserved_special_token_207|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128216": {
+      "content": "<|reserved_special_token_208|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128217": {
+      "content": "<|reserved_special_token_209|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128218": {
+      "content": "<|reserved_special_token_210|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128219": {
+      "content": "<|reserved_special_token_211|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128220": {
+      "content": "<|reserved_special_token_212|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128221": {
+      "content": "<|reserved_special_token_213|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128222": {
+      "content": "<|reserved_special_token_214|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128223": {
+      "content": "<|reserved_special_token_215|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128224": {
+      "content": "<|reserved_special_token_216|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128225": {
+      "content": "<|reserved_special_token_217|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128226": {
+      "content": "<|reserved_special_token_218|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128227": {
+      "content": "<|reserved_special_token_219|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128228": {
+      "content": "<|reserved_special_token_220|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128229": {
+      "content": "<|reserved_special_token_221|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128230": {
+      "content": "<|reserved_special_token_222|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128231": {
+      "content": "<|reserved_special_token_223|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128232": {
+      "content": "<|reserved_special_token_224|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128233": {
+      "content": "<|reserved_special_token_225|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128234": {
+      "content": "<|reserved_special_token_226|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128235": {
+      "content": "<|reserved_special_token_227|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128236": {
+      "content": "<|reserved_special_token_228|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128237": {
+      "content": "<|reserved_special_token_229|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128238": {
+      "content": "<|reserved_special_token_230|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128239": {
+      "content": "<|reserved_special_token_231|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128240": {
+      "content": "<|reserved_special_token_232|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128241": {
+      "content": "<|reserved_special_token_233|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128242": {
+      "content": "<|reserved_special_token_234|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128243": {
+      "content": "<|reserved_special_token_235|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128244": {
+      "content": "<|reserved_special_token_236|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128245": {
+      "content": "<|reserved_special_token_237|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128246": {
+      "content": "<|reserved_special_token_238|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128247": {
+      "content": "<|reserved_special_token_239|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128248": {
+      "content": "<|reserved_special_token_240|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128249": {
+      "content": "<|reserved_special_token_241|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128250": {
+      "content": "<|reserved_special_token_242|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128251": {
+      "content": "<|reserved_special_token_243|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128252": {
+      "content": "<|reserved_special_token_244|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128253": {
+      "content": "<|reserved_special_token_245|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128254": {
+      "content": "<|reserved_special_token_246|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128255": {
+      "content": "<|reserved_special_token_247|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|begin_of_text|>",
+  "chat_template": "{{- bos_token }}{%- if messages[0]['role'] == 'system' %}{%- set system_message = messages[0]['content']|trim %}{%- set messages = messages[1:] %}{%- else %}{%- set system_message = \"\" %}{%- endif %}{{- \"<|start_header_id|>system<|end_header_id|>\\n\\n\" }}{{- system_message }}{{- \"<|eot_id|>\" }}{%- for message in messages %}{%- if message['role'] == 'assistant' and '</think>' in message['content'] %}{%- set content = message['content'].split('</think>')[-1].lstrip() %}{%- else %}{%- set content = message['content'] %}{%- endif %}{{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\\n\\n' + content | trim + '<|eot_id|>' }}{%- endfor %}{%- if add_generation_prompt %}{{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' }}{%- endif %}",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|eot_id|>",
+  "extra_special_tokens": {},
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "pad_token": "<|end_of_text|>",
+  "tokenizer_class": "PreTrainedTokenizer"
+}
diff --git a/checkpoint-436/trainer_state.json b/checkpoint-436/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..83b071f85f3c16fe907c840f1d52df50520a5b99
--- /dev/null
+++ b/checkpoint-436/trainer_state.json
@@ -0,0 +1,3085 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.9695121951219514,
+  "eval_steps": 500,
+  "global_step": 436,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.009146341463414634,
+      "grad_norm": 40.05127716064453,
+      "learning_rate": 5.0000000000000004e-08,
+      "loss": 5.1282,
+      "step": 1
+    },
+    {
+      "epoch": 0.018292682926829267,
+      "grad_norm": 39.01039505004883,
+      "learning_rate": 1.0000000000000001e-07,
+      "loss": 5.0586,
+      "step": 2
+    },
+    {
+      "epoch": 0.027439024390243903,
+      "grad_norm": 40.20766067504883,
+      "learning_rate": 1.5000000000000002e-07,
+      "loss": 5.0311,
+      "step": 3
+    },
+    {
+      "epoch": 0.036585365853658534,
+      "grad_norm": 38.70350646972656,
+      "learning_rate": 2.0000000000000002e-07,
+      "loss": 5.1598,
+      "step": 4
+    },
+    {
+      "epoch": 0.04573170731707317,
+      "grad_norm": 39.230979919433594,
+      "learning_rate": 2.5000000000000004e-07,
+      "loss": 5.0812,
+      "step": 5
+    },
+    {
+      "epoch": 0.054878048780487805,
+      "grad_norm": 39.45293045043945,
+      "learning_rate": 3.0000000000000004e-07,
+      "loss": 5.0841,
+      "step": 6
+    },
+    {
+      "epoch": 0.06402439024390244,
+      "grad_norm": 38.92170333862305,
+      "learning_rate": 3.5000000000000004e-07,
+      "loss": 5.022,
+      "step": 7
+    },
+    {
+      "epoch": 0.07317073170731707,
+      "grad_norm": 38.932369232177734,
+      "learning_rate": 4.0000000000000003e-07,
+      "loss": 5.0876,
+      "step": 8
+    },
+    {
+      "epoch": 0.08231707317073171,
+      "grad_norm": 39.60245132446289,
+      "learning_rate": 4.5000000000000003e-07,
+      "loss": 5.0798,
+      "step": 9
+    },
+    {
+      "epoch": 0.09146341463414634,
+      "grad_norm": 39.53607177734375,
+      "learning_rate": 5.000000000000001e-07,
+      "loss": 5.0105,
+      "step": 10
+    },
+    {
+      "epoch": 0.10060975609756098,
+      "grad_norm": 39.50700378417969,
+      "learning_rate": 5.5e-07,
+      "loss": 4.9951,
+      "step": 11
+    },
+    {
+      "epoch": 0.10975609756097561,
+      "grad_norm": 39.52376174926758,
+      "learning_rate": 6.000000000000001e-07,
+      "loss": 4.9163,
+      "step": 12
+    },
+    {
+      "epoch": 0.11890243902439024,
+      "grad_norm": 39.15214157104492,
+      "learning_rate": 6.5e-07,
+      "loss": 4.9634,
+      "step": 13
+    },
+    {
+      "epoch": 0.12804878048780488,
+      "grad_norm": 37.795066833496094,
+      "learning_rate": 7.000000000000001e-07,
+      "loss": 4.819,
+      "step": 14
+    },
+    {
+      "epoch": 0.13719512195121952,
+      "grad_norm": 38.23014450073242,
+      "learning_rate": 7.5e-07,
+      "loss": 4.8756,
+      "step": 15
+    },
+    {
+      "epoch": 0.14634146341463414,
+      "grad_norm": 38.57551574707031,
+      "learning_rate": 8.000000000000001e-07,
+      "loss": 4.81,
+      "step": 16
+    },
+    {
+      "epoch": 0.15548780487804878,
+      "grad_norm": 36.06121063232422,
+      "learning_rate": 8.500000000000001e-07,
+      "loss": 4.7901,
+      "step": 17
+    },
+    {
+      "epoch": 0.16463414634146342,
+      "grad_norm": 34.38008117675781,
+      "learning_rate": 9.000000000000001e-07,
+      "loss": 4.5775,
+      "step": 18
+    },
+    {
+      "epoch": 0.17378048780487804,
+      "grad_norm": 32.60890579223633,
+      "learning_rate": 9.500000000000001e-07,
+      "loss": 4.3737,
+      "step": 19
+    },
+    {
+      "epoch": 0.18292682926829268,
+      "grad_norm": 31.983110427856445,
+      "learning_rate": 1.0000000000000002e-06,
+      "loss": 4.3773,
+      "step": 20
+    },
+    {
+      "epoch": 0.19207317073170732,
+      "grad_norm": 27.69571876525879,
+      "learning_rate": 1.0500000000000001e-06,
+      "loss": 4.1276,
+      "step": 21
+    },
+    {
+      "epoch": 0.20121951219512196,
+      "grad_norm": 26.542314529418945,
+      "learning_rate": 1.1e-06,
+      "loss": 4.0641,
+      "step": 22
+    },
+    {
+      "epoch": 0.21036585365853658,
+      "grad_norm": 23.26603126525879,
+      "learning_rate": 1.1500000000000002e-06,
+      "loss": 4.0197,
+      "step": 23
+    },
+    {
+      "epoch": 0.21951219512195122,
+      "grad_norm": 21.04385757446289,
+      "learning_rate": 1.2000000000000002e-06,
+      "loss": 4.0179,
+      "step": 24
+    },
+    {
+      "epoch": 0.22865853658536586,
+      "grad_norm": 19.790103912353516,
+      "learning_rate": 1.25e-06,
+      "loss": 3.8334,
+      "step": 25
+    },
+    {
+      "epoch": 0.23780487804878048,
+      "grad_norm": 18.2396183013916,
+      "learning_rate": 1.3e-06,
+      "loss": 3.7027,
+      "step": 26
+    },
+    {
+      "epoch": 0.24695121951219512,
+      "grad_norm": 18.725860595703125,
+      "learning_rate": 1.3500000000000002e-06,
+      "loss": 3.7268,
+      "step": 27
+    },
+    {
+      "epoch": 0.25609756097560976,
+      "grad_norm": 18.91561508178711,
+      "learning_rate": 1.4000000000000001e-06,
+      "loss": 3.52,
+      "step": 28
+    },
+    {
+      "epoch": 0.2652439024390244,
+      "grad_norm": 18.196651458740234,
+      "learning_rate": 1.45e-06,
+      "loss": 3.4765,
+      "step": 29
+    },
+    {
+      "epoch": 0.27439024390243905,
+      "grad_norm": 17.778606414794922,
+      "learning_rate": 1.5e-06,
+      "loss": 3.3573,
+      "step": 30
+    },
+    {
+      "epoch": 0.28353658536585363,
+      "grad_norm": 16.23012924194336,
+      "learning_rate": 1.5500000000000002e-06,
+      "loss": 3.2783,
+      "step": 31
+    },
+    {
+      "epoch": 0.2926829268292683,
+      "grad_norm": 15.836565017700195,
+      "learning_rate": 1.6000000000000001e-06,
+      "loss": 3.2319,
+      "step": 32
+    },
+    {
+      "epoch": 0.3018292682926829,
+      "grad_norm": 11.919289588928223,
+      "learning_rate": 1.6500000000000003e-06,
+      "loss": 3.1345,
+      "step": 33
+    },
+    {
+      "epoch": 0.31097560975609756,
+      "grad_norm": 11.086843490600586,
+      "learning_rate": 1.7000000000000002e-06,
+      "loss": 3.1153,
+      "step": 34
+    },
+    {
+      "epoch": 0.3201219512195122,
+      "grad_norm": 7.563486576080322,
+      "learning_rate": 1.75e-06,
+      "loss": 2.9784,
+      "step": 35
+    },
+    {
+      "epoch": 0.32926829268292684,
+      "grad_norm": 7.60545539855957,
+      "learning_rate": 1.8000000000000001e-06,
+      "loss": 2.8568,
+      "step": 36
+    },
+    {
+      "epoch": 0.3384146341463415,
+      "grad_norm": 7.071218013763428,
+      "learning_rate": 1.85e-06,
+      "loss": 2.8879,
+      "step": 37
+    },
+    {
+      "epoch": 0.3475609756097561,
+      "grad_norm": 8.200672149658203,
+      "learning_rate": 1.9000000000000002e-06,
+      "loss": 2.7437,
+      "step": 38
+    },
+    {
+      "epoch": 0.3567073170731707,
+      "grad_norm": 12.079374313354492,
+      "learning_rate": 1.9500000000000004e-06,
+      "loss": 2.7589,
+      "step": 39
+    },
+    {
+      "epoch": 0.36585365853658536,
+      "grad_norm": 12.21938705444336,
+      "learning_rate": 2.0000000000000003e-06,
+      "loss": 2.7098,
+      "step": 40
+    },
+    {
+      "epoch": 0.375,
+      "grad_norm": 10.224228858947754,
+      "learning_rate": 2.05e-06,
+      "loss": 2.6803,
+      "step": 41
+    },
+    {
+      "epoch": 0.38414634146341464,
+      "grad_norm": 10.054616928100586,
+      "learning_rate": 2.1000000000000002e-06,
+      "loss": 2.4658,
+      "step": 42
+    },
+    {
+      "epoch": 0.3932926829268293,
+      "grad_norm": 8.074049949645996,
+      "learning_rate": 2.15e-06,
+      "loss": 2.6341,
+      "step": 43
+    },
+    {
+      "epoch": 0.4024390243902439,
+      "grad_norm": 5.721987724304199,
+      "learning_rate": 2.2e-06,
+      "loss": 2.3848,
+      "step": 44
+    },
+    {
+      "epoch": 0.4115853658536585,
+      "grad_norm": 4.646566390991211,
+      "learning_rate": 2.25e-06,
+      "loss": 2.4323,
+      "step": 45
+    },
+    {
+      "epoch": 0.42073170731707316,
+      "grad_norm": 4.591104984283447,
+      "learning_rate": 2.3000000000000004e-06,
+      "loss": 2.4022,
+      "step": 46
+    },
+    {
+      "epoch": 0.4298780487804878,
+      "grad_norm": 4.822715759277344,
+      "learning_rate": 2.35e-06,
+      "loss": 2.2768,
+      "step": 47
+    },
+    {
+      "epoch": 0.43902439024390244,
+      "grad_norm": 5.176568984985352,
+      "learning_rate": 2.4000000000000003e-06,
+      "loss": 2.281,
+      "step": 48
+    },
+    {
+      "epoch": 0.4481707317073171,
+      "grad_norm": 4.731242656707764,
+      "learning_rate": 2.4500000000000003e-06,
+      "loss": 2.235,
+      "step": 49
+    },
+    {
+      "epoch": 0.4573170731707317,
+      "grad_norm": 4.6343865394592285,
+      "learning_rate": 2.5e-06,
+      "loss": 2.1465,
+      "step": 50
+    },
+    {
+      "epoch": 0.46646341463414637,
+      "grad_norm": 4.669382095336914,
+      "learning_rate": 2.55e-06,
+      "loss": 2.0969,
+      "step": 51
+    },
+    {
+      "epoch": 0.47560975609756095,
+      "grad_norm": 4.207769870758057,
+      "learning_rate": 2.6e-06,
+      "loss": 2.0308,
+      "step": 52
+    },
+    {
+      "epoch": 0.4847560975609756,
+      "grad_norm": 5.222470760345459,
+      "learning_rate": 2.6500000000000005e-06,
+      "loss": 1.9445,
+      "step": 53
+    },
+    {
+      "epoch": 0.49390243902439024,
+      "grad_norm": 5.10925817489624,
+      "learning_rate": 2.7000000000000004e-06,
+      "loss": 1.7763,
+      "step": 54
+    },
+    {
+      "epoch": 0.5030487804878049,
+      "grad_norm": 4.8151044845581055,
+      "learning_rate": 2.7500000000000004e-06,
+      "loss": 1.7661,
+      "step": 55
+    },
+    {
+      "epoch": 0.5121951219512195,
+      "grad_norm": 6.155700206756592,
+      "learning_rate": 2.8000000000000003e-06,
+      "loss": 1.6665,
+      "step": 56
+    },
+    {
+      "epoch": 0.5213414634146342,
+      "grad_norm": 4.719021320343018,
+      "learning_rate": 2.85e-06,
+      "loss": 1.5532,
+      "step": 57
+    },
+    {
+      "epoch": 0.5304878048780488,
+      "grad_norm": 4.591799259185791,
+      "learning_rate": 2.9e-06,
+      "loss": 1.5244,
+      "step": 58
+    },
+    {
+      "epoch": 0.5396341463414634,
+      "grad_norm": 5.483795642852783,
+      "learning_rate": 2.95e-06,
+      "loss": 1.3816,
+      "step": 59
+    },
+    {
+      "epoch": 0.5487804878048781,
+      "grad_norm": 4.32553768157959,
+      "learning_rate": 3e-06,
+      "loss": 1.4268,
+      "step": 60
+    },
+    {
+      "epoch": 0.5579268292682927,
+      "grad_norm": 8.296231269836426,
+      "learning_rate": 3.05e-06,
+      "loss": 1.2887,
+      "step": 61
+    },
+    {
+      "epoch": 0.5670731707317073,
+      "grad_norm": 5.07594108581543,
+      "learning_rate": 3.1000000000000004e-06,
+      "loss": 1.228,
+      "step": 62
+    },
+    {
+      "epoch": 0.5762195121951219,
+      "grad_norm": 8.427435874938965,
+      "learning_rate": 3.1500000000000003e-06,
+      "loss": 1.1724,
+      "step": 63
+    },
+    {
+      "epoch": 0.5853658536585366,
+      "grad_norm": 22.45092010498047,
+      "learning_rate": 3.2000000000000003e-06,
+      "loss": 1.0871,
+      "step": 64
+    },
+    {
+      "epoch": 0.5945121951219512,
+      "grad_norm": 20.67412567138672,
+      "learning_rate": 3.2500000000000002e-06,
+      "loss": 1.0108,
+      "step": 65
+    },
+    {
+      "epoch": 0.6036585365853658,
+      "grad_norm": 4.921634674072266,
+      "learning_rate": 3.3000000000000006e-06,
+      "loss": 0.9775,
+      "step": 66
+    },
+    {
+      "epoch": 0.6128048780487805,
+      "grad_norm": 4.477286338806152,
+      "learning_rate": 3.3500000000000005e-06,
+      "loss": 0.9554,
+      "step": 67
+    },
+    {
+      "epoch": 0.6219512195121951,
+      "grad_norm": 8.190194129943848,
+      "learning_rate": 3.4000000000000005e-06,
+      "loss": 0.8583,
+      "step": 68
+    },
+    {
+      "epoch": 0.6310975609756098,
+      "grad_norm": 29.21201515197754,
+      "learning_rate": 3.45e-06,
+      "loss": 0.8499,
+      "step": 69
+    },
+    {
+      "epoch": 0.6402439024390244,
+      "grad_norm": 33.81047439575195,
+      "learning_rate": 3.5e-06,
+      "loss": 0.858,
+      "step": 70
+    },
+    {
+      "epoch": 0.649390243902439,
+      "grad_norm": 3.5125606060028076,
+      "learning_rate": 3.5500000000000003e-06,
+      "loss": 0.7227,
+      "step": 71
+    },
+    {
+      "epoch": 0.6585365853658537,
+      "grad_norm": 18.493423461914062,
+      "learning_rate": 3.6000000000000003e-06,
+      "loss": 0.7447,
+      "step": 72
+    },
+    {
+      "epoch": 0.6676829268292683,
+      "grad_norm": 24.923927307128906,
+      "learning_rate": 3.65e-06,
+      "loss": 0.8423,
+      "step": 73
+    },
+    {
+      "epoch": 0.676829268292683,
+      "grad_norm": 9.870631217956543,
+      "learning_rate": 3.7e-06,
+      "loss": 0.6934,
+      "step": 74
+    },
+    {
+      "epoch": 0.6859756097560976,
+      "grad_norm": 20.598569869995117,
+      "learning_rate": 3.7500000000000005e-06,
+      "loss": 0.7005,
+      "step": 75
+    },
+    {
+      "epoch": 0.6951219512195121,
+      "grad_norm": 30.662466049194336,
+      "learning_rate": 3.8000000000000005e-06,
+      "loss": 0.6226,
+      "step": 76
+    },
+    {
+      "epoch": 0.7042682926829268,
+      "grad_norm": 28.930774688720703,
+      "learning_rate": 3.85e-06,
+      "loss": 0.6357,
+      "step": 77
+    },
+    {
+      "epoch": 0.7134146341463414,
+      "grad_norm": 7.140467166900635,
+      "learning_rate": 3.900000000000001e-06,
+      "loss": 0.6129,
+      "step": 78
+    },
+    {
+      "epoch": 0.7225609756097561,
+      "grad_norm": 6.322911739349365,
+      "learning_rate": 3.95e-06,
+      "loss": 0.6086,
+      "step": 79
+    },
+    {
+      "epoch": 0.7317073170731707,
+      "grad_norm": 5.404141426086426,
+      "learning_rate": 4.000000000000001e-06,
+      "loss": 0.5986,
+      "step": 80
+    },
+    {
+      "epoch": 0.7408536585365854,
+      "grad_norm": 24.445228576660156,
+      "learning_rate": 4.05e-06,
+      "loss": 0.5932,
+      "step": 81
+    },
+    {
+      "epoch": 0.75,
+      "grad_norm": 14.275381088256836,
+      "learning_rate": 4.1e-06,
+      "loss": 0.5286,
+      "step": 82
+    },
+    {
+      "epoch": 0.7591463414634146,
+      "grad_norm": 9.77132511138916,
+      "learning_rate": 4.15e-06,
+      "loss": 0.4814,
+      "step": 83
+    },
+    {
+      "epoch": 0.7682926829268293,
+      "grad_norm": 11.043930053710938,
+      "learning_rate": 4.2000000000000004e-06,
+      "loss": 0.4953,
+      "step": 84
+    },
+    {
+      "epoch": 0.7774390243902439,
+      "grad_norm": 3.1333775520324707,
+      "learning_rate": 4.25e-06,
+      "loss": 0.4794,
+      "step": 85
+    },
+    {
+      "epoch": 0.7865853658536586,
+      "grad_norm": 9.50448989868164,
+      "learning_rate": 4.3e-06,
+      "loss": 0.4451,
+      "step": 86
+    },
+    {
+      "epoch": 0.7957317073170732,
+      "grad_norm": 5.867074489593506,
+      "learning_rate": 4.350000000000001e-06,
+      "loss": 0.4224,
+      "step": 87
+    },
+    {
+      "epoch": 0.8048780487804879,
+      "grad_norm": 12.556106567382812,
+      "learning_rate": 4.4e-06,
+      "loss": 0.4224,
+      "step": 88
+    },
+    {
+      "epoch": 0.8140243902439024,
+      "grad_norm": 7.916622638702393,
+      "learning_rate": 4.450000000000001e-06,
+      "loss": 0.4091,
+      "step": 89
+    },
+    {
+      "epoch": 0.823170731707317,
+      "grad_norm": 2.970932722091675,
+      "learning_rate": 4.5e-06,
+      "loss": 0.4181,
+      "step": 90
+    },
+    {
+      "epoch": 0.8323170731707317,
+      "grad_norm": 17.468870162963867,
+      "learning_rate": 4.5500000000000005e-06,
+      "loss": 0.4031,
+      "step": 91
+    },
+    {
+      "epoch": 0.8414634146341463,
+      "grad_norm": 15.861083984375,
+      "learning_rate": 4.600000000000001e-06,
+      "loss": 0.3604,
+      "step": 92
+    },
+    {
+      "epoch": 0.850609756097561,
+      "grad_norm": 6.715958118438721,
+      "learning_rate": 4.65e-06,
+      "loss": 0.3489,
+      "step": 93
+    },
+    {
+      "epoch": 0.8597560975609756,
+      "grad_norm": 5.439844608306885,
+      "learning_rate": 4.7e-06,
+      "loss": 0.3339,
+      "step": 94
+    },
+    {
+      "epoch": 0.8689024390243902,
+      "grad_norm": 4.650434494018555,
+      "learning_rate": 4.75e-06,
+      "loss": 0.3189,
+      "step": 95
+    },
+    {
+      "epoch": 0.8780487804878049,
+      "grad_norm": 15.319528579711914,
+      "learning_rate": 4.800000000000001e-06,
+      "loss": 0.3347,
+      "step": 96
+    },
+    {
+      "epoch": 0.8871951219512195,
+      "grad_norm": 5.151925563812256,
+      "learning_rate": 4.85e-06,
+      "loss": 0.3043,
+      "step": 97
+    },
+    {
+      "epoch": 0.8963414634146342,
+      "grad_norm": 6.351178169250488,
+      "learning_rate": 4.9000000000000005e-06,
+      "loss": 0.2939,
+      "step": 98
+    },
+    {
+      "epoch": 0.9054878048780488,
+      "grad_norm": 3.942911148071289,
+      "learning_rate": 4.95e-06,
+      "loss": 0.2887,
+      "step": 99
+    },
+    {
+      "epoch": 0.9146341463414634,
+      "grad_norm": 3.0885493755340576,
+      "learning_rate": 5e-06,
+      "loss": 0.2992,
+      "step": 100
+    },
+    {
+      "epoch": 0.9237804878048781,
+      "grad_norm": 7.023274898529053,
+      "learning_rate": 4.999959803423605e-06,
+      "loss": 0.2607,
+      "step": 101
+    },
+    {
+      "epoch": 0.9329268292682927,
+      "grad_norm": 8.774216651916504,
+      "learning_rate": 4.9998392149870315e-06,
+      "loss": 0.2674,
+      "step": 102
+    },
+    {
+      "epoch": 0.9420731707317073,
+      "grad_norm": 3.5439791679382324,
+      "learning_rate": 4.999638238568072e-06,
+      "loss": 0.2602,
+      "step": 103
+    },
+    {
+      "epoch": 0.9512195121951219,
+      "grad_norm": 7.669381618499756,
+      "learning_rate": 4.999356880629579e-06,
+      "loss": 0.2718,
+      "step": 104
+    },
+    {
+      "epoch": 0.9603658536585366,
+      "grad_norm": 6.9407854080200195,
+      "learning_rate": 4.998995150219252e-06,
+      "loss": 0.2494,
+      "step": 105
+    },
+    {
+      "epoch": 0.9695121951219512,
+      "grad_norm": 12.144411087036133,
+      "learning_rate": 4.998553058969352e-06,
+      "loss": 0.2297,
+      "step": 106
+    },
+    {
+      "epoch": 0.9786585365853658,
+      "grad_norm": 6.956138610839844,
+      "learning_rate": 4.998030621096322e-06,
+      "loss": 0.2386,
+      "step": 107
+    },
+    {
+      "epoch": 0.9878048780487805,
+      "grad_norm": 8.304037094116211,
+      "learning_rate": 4.997427853400333e-06,
+      "loss": 0.23,
+      "step": 108
+    },
+    {
+      "epoch": 0.9969512195121951,
+      "grad_norm": 7.031365394592285,
+      "learning_rate": 4.996744775264743e-06,
+      "loss": 0.2346,
+      "step": 109
+    },
+    {
+      "epoch": 1.0,
+      "grad_norm": 7.031365394592285,
+      "learning_rate": 4.9959814086554735e-06,
+      "loss": 0.0739,
+      "step": 110
+    },
+    {
+      "epoch": 1.0091463414634145,
+      "grad_norm": 2.084608793258667,
+      "learning_rate": 4.995137778120306e-06,
+      "loss": 0.2422,
+      "step": 111
+    },
+    {
+      "epoch": 1.0182926829268293,
+      "grad_norm": 38.12245178222656,
+      "learning_rate": 4.994213910788085e-06,
+      "loss": 0.1992,
+      "step": 112
+    },
+    {
+      "epoch": 1.0274390243902438,
+      "grad_norm": 2.817565441131592,
+      "learning_rate": 4.993209836367856e-06,
+      "loss": 0.1882,
+      "step": 113
+    },
+    {
+      "epoch": 1.0365853658536586,
+      "grad_norm": 11.774249076843262,
+      "learning_rate": 4.9921255871479e-06,
+      "loss": 0.2235,
+      "step": 114
+    },
+    {
+      "epoch": 1.045731707317073,
+      "grad_norm": 2.3362019062042236,
+      "learning_rate": 4.9909611979947046e-06,
+      "loss": 0.1875,
+      "step": 115
+    },
+    {
+      "epoch": 1.0548780487804879,
+      "grad_norm": 1.9516818523406982,
+      "learning_rate": 4.989716706351835e-06,
+      "loss": 0.1858,
+      "step": 116
+    },
+    {
+      "epoch": 1.0640243902439024,
+      "grad_norm": 1.5484645366668701,
+      "learning_rate": 4.9883921522387336e-06,
+      "loss": 0.1845,
+      "step": 117
+    },
+    {
+      "epoch": 1.0731707317073171,
+      "grad_norm": 7.430396556854248,
+      "learning_rate": 4.986987578249433e-06,
+      "loss": 0.184,
+      "step": 118
+    },
+    {
+      "epoch": 1.0823170731707317,
+      "grad_norm": 2.0725948810577393,
+      "learning_rate": 4.9855030295511845e-06,
+      "loss": 0.177,
+      "step": 119
+    },
+    {
+      "epoch": 1.0914634146341464,
+      "grad_norm": 4.836520671844482,
+      "learning_rate": 4.98393855388301e-06,
+      "loss": 0.1744,
+      "step": 120
+    },
+    {
+      "epoch": 1.100609756097561,
+      "grad_norm": 5.22581672668457,
+      "learning_rate": 4.982294201554162e-06,
+      "loss": 0.1599,
+      "step": 121
+    },
+    {
+      "epoch": 1.1097560975609757,
+      "grad_norm": 5.406208038330078,
+      "learning_rate": 4.980570025442507e-06,
+      "loss": 0.1645,
+      "step": 122
+    },
+    {
+      "epoch": 1.1189024390243902,
+      "grad_norm": 8.736640930175781,
+      "learning_rate": 4.978766080992827e-06,
+      "loss": 0.1943,
+      "step": 123
+    },
+    {
+      "epoch": 1.1280487804878048,
+      "grad_norm": 9.184198379516602,
+      "learning_rate": 4.976882426215035e-06,
+      "loss": 0.1753,
+      "step": 124
+    },
+    {
+      "epoch": 1.1371951219512195,
+      "grad_norm": 5.317800521850586,
+      "learning_rate": 4.974919121682307e-06,
+      "loss": 0.1897,
+      "step": 125
+    },
+    {
+      "epoch": 1.146341463414634,
+      "grad_norm": 11.803364753723145,
+      "learning_rate": 4.972876230529143e-06,
+      "loss": 0.1516,
+      "step": 126
+    },
+    {
+      "epoch": 1.1554878048780488,
+      "grad_norm": 31.74310302734375,
+      "learning_rate": 4.9707538184493255e-06,
+      "loss": 0.1947,
+      "step": 127
+    },
+    {
+      "epoch": 1.1646341463414633,
+      "grad_norm": 1.956932783126831,
+      "learning_rate": 4.968551953693813e-06,
+      "loss": 0.1708,
+      "step": 128
+    },
+    {
+      "epoch": 1.173780487804878,
+      "grad_norm": 4.64445686340332,
+      "learning_rate": 4.966270707068548e-06,
+      "loss": 0.1711,
+      "step": 129
+    },
+    {
+      "epoch": 1.1829268292682926,
+      "grad_norm": 4.442819595336914,
+      "learning_rate": 4.963910151932171e-06,
+      "loss": 0.1572,
+      "step": 130
+    },
+    {
+      "epoch": 1.1920731707317074,
+      "grad_norm": 1.9702759981155396,
+      "learning_rate": 4.961470364193673e-06,
+      "loss": 0.1364,
+      "step": 131
+    },
+    {
+      "epoch": 1.201219512195122,
+      "grad_norm": 7.816587448120117,
+      "learning_rate": 4.9589514223099425e-06,
+      "loss": 0.1624,
+      "step": 132
+    },
+    {
+      "epoch": 1.2103658536585367,
+      "grad_norm": 61.18424606323242,
+      "learning_rate": 4.956353407283252e-06,
+      "loss": 0.26,
+      "step": 133
+    },
+    {
+      "epoch": 1.2195121951219512,
+      "grad_norm": 15.649847984313965,
+      "learning_rate": 4.9536764026586505e-06,
+      "loss": 0.1543,
+      "step": 134
+    },
+    {
+      "epoch": 1.228658536585366,
+      "grad_norm": 2.249363660812378,
+      "learning_rate": 4.950920494521274e-06,
+      "loss": 0.152,
+      "step": 135
+    },
+    {
+      "epoch": 1.2378048780487805,
+      "grad_norm": 4.055661678314209,
+      "learning_rate": 4.948085771493579e-06,
+      "loss": 0.1402,
+      "step": 136
+    },
+    {
+      "epoch": 1.2469512195121952,
+      "grad_norm": 2.2803292274475098,
+      "learning_rate": 4.945172324732496e-06,
+      "loss": 0.1427,
+      "step": 137
+    },
+    {
+      "epoch": 1.2560975609756098,
+      "grad_norm": 2.5540456771850586,
+      "learning_rate": 4.942180247926492e-06,
+      "loss": 0.143,
+      "step": 138
+    },
+    {
+      "epoch": 1.2652439024390243,
+      "grad_norm": 8.80340576171875,
+      "learning_rate": 4.939109637292563e-06,
+      "loss": 0.1569,
+      "step": 139
+    },
+    {
+      "epoch": 1.274390243902439,
+      "grad_norm": 3.1077523231506348,
+      "learning_rate": 4.935960591573136e-06,
+      "loss": 0.1313,
+      "step": 140
+    },
+    {
+      "epoch": 1.2835365853658536,
+      "grad_norm": 1.2506815195083618,
+      "learning_rate": 4.932733212032896e-06,
+      "loss": 0.137,
+      "step": 141
+    },
+    {
+      "epoch": 1.2926829268292683,
+      "grad_norm": 2.4397666454315186,
+      "learning_rate": 4.929427602455532e-06,
+      "loss": 0.1408,
+      "step": 142
+    },
+    {
+      "epoch": 1.3018292682926829,
+      "grad_norm": 1.7658663988113403,
+      "learning_rate": 4.926043869140392e-06,
+      "loss": 0.133,
+      "step": 143
+    },
+    {
+      "epoch": 1.3109756097560976,
+      "grad_norm": 1.0473108291625977,
+      "learning_rate": 4.922582120899072e-06,
+      "loss": 0.1358,
+      "step": 144
+    },
+    {
+      "epoch": 1.3201219512195121,
+      "grad_norm": 5.566607475280762,
+      "learning_rate": 4.919042469051916e-06,
+      "loss": 0.1237,
+      "step": 145
+    },
+    {
+      "epoch": 1.329268292682927,
+      "grad_norm": 1.0360139608383179,
+      "learning_rate": 4.915425027424429e-06,
+      "loss": 0.1274,
+      "step": 146
+    },
+    {
+      "epoch": 1.3384146341463414,
+      "grad_norm": 1.9091176986694336,
+      "learning_rate": 4.911729912343631e-06,
+      "loss": 0.1439,
+      "step": 147
+    },
+    {
+      "epoch": 1.3475609756097562,
+      "grad_norm": 1.5727638006210327,
+      "learning_rate": 4.907957242634299e-06,
+      "loss": 0.1259,
+      "step": 148
+    },
+    {
+      "epoch": 1.3567073170731707,
+      "grad_norm": 1.0574792623519897,
+      "learning_rate": 4.9041071396151585e-06,
+      "loss": 0.142,
+      "step": 149
+    },
+    {
+      "epoch": 1.3658536585365852,
+      "grad_norm": 8.020130157470703,
+      "learning_rate": 4.900179727094978e-06,
+      "loss": 0.1392,
+      "step": 150
+    },
+    {
+      "epoch": 1.375,
+      "grad_norm": 3.156255006790161,
+      "learning_rate": 4.896175131368587e-06,
+      "loss": 0.152,
+      "step": 151
+    },
+    {
+      "epoch": 1.3841463414634148,
+      "grad_norm": 2.3956990242004395,
+      "learning_rate": 4.892093481212817e-06,
+      "loss": 0.1244,
+      "step": 152
+    },
+    {
+      "epoch": 1.3932926829268293,
+      "grad_norm": 2.6058578491210938,
+      "learning_rate": 4.887934907882357e-06,
+      "loss": 0.1271,
+      "step": 153
+    },
+    {
+      "epoch": 1.4024390243902438,
+      "grad_norm": 1.7173289060592651,
+      "learning_rate": 4.883699545105535e-06,
+      "loss": 0.1257,
+      "step": 154
+    },
+    {
+      "epoch": 1.4115853658536586,
+      "grad_norm": 2.2237367630004883,
+      "learning_rate": 4.8793875290800186e-06,
+      "loss": 0.127,
+      "step": 155
+    },
+    {
+      "epoch": 1.420731707317073,
+      "grad_norm": 6.997623920440674,
+      "learning_rate": 4.874998998468433e-06,
+      "loss": 0.1454,
+      "step": 156
+    },
+    {
+      "epoch": 1.4298780487804879,
+      "grad_norm": 10.645139694213867,
+      "learning_rate": 4.870534094393903e-06,
+      "loss": 0.1402,
+      "step": 157
+    },
+    {
+      "epoch": 1.4390243902439024,
+      "grad_norm": 1.5808072090148926,
+      "learning_rate": 4.865992960435514e-06,
+      "loss": 0.1156,
+      "step": 158
+    },
+    {
+      "epoch": 1.4481707317073171,
+      "grad_norm": 0.990803062915802,
+      "learning_rate": 4.861375742623697e-06,
+      "loss": 0.1286,
+      "step": 159
+    },
+    {
+      "epoch": 1.4573170731707317,
+      "grad_norm": 1.5008662939071655,
+      "learning_rate": 4.856682589435531e-06,
+      "loss": 0.1282,
+      "step": 160
+    },
+    {
+      "epoch": 1.4664634146341464,
+      "grad_norm": 1.772304892539978,
+      "learning_rate": 4.851913651789968e-06,
+      "loss": 0.1308,
+      "step": 161
+    },
+    {
+      "epoch": 1.475609756097561,
+      "grad_norm": 1.201061487197876,
+      "learning_rate": 4.847069083042982e-06,
+      "loss": 0.1198,
+      "step": 162
+    },
+    {
+      "epoch": 1.4847560975609757,
+      "grad_norm": 0.6147828698158264,
+      "learning_rate": 4.8421490389826344e-06,
+      "loss": 0.1138,
+      "step": 163
+    },
+    {
+      "epoch": 1.4939024390243902,
+      "grad_norm": 1.3990256786346436,
+      "learning_rate": 4.837153677824068e-06,
+      "loss": 0.1205,
+      "step": 164
+    },
+    {
+      "epoch": 1.5030487804878048,
+      "grad_norm": 1.6977574825286865,
+      "learning_rate": 4.832083160204414e-06,
+      "loss": 0.1128,
+      "step": 165
+    },
+    {
+      "epoch": 1.5121951219512195,
+      "grad_norm": 2.4056453704833984,
+      "learning_rate": 4.826937649177632e-06,
+      "loss": 0.1184,
+      "step": 166
+    },
+    {
+      "epoch": 1.5213414634146343,
+      "grad_norm": 2.7455694675445557,
+      "learning_rate": 4.821717310209265e-06,
+      "loss": 0.123,
+      "step": 167
+    },
+    {
+      "epoch": 1.5304878048780488,
+      "grad_norm": 1.8243584632873535,
+      "learning_rate": 4.816422311171115e-06,
+      "loss": 0.1198,
+      "step": 168
+    },
+    {
+      "epoch": 1.5396341463414633,
+      "grad_norm": 0.8920577764511108,
+      "learning_rate": 4.811052822335849e-06,
+      "loss": 0.1088,
+      "step": 169
+    },
+    {
+      "epoch": 1.548780487804878,
+      "grad_norm": 1.6639117002487183,
+      "learning_rate": 4.805609016371522e-06,
+      "loss": 0.1231,
+      "step": 170
+    },
+    {
+      "epoch": 1.5579268292682928,
+      "grad_norm": 1.383026123046875,
+      "learning_rate": 4.800091068336024e-06,
+      "loss": 0.1207,
+      "step": 171
+    },
+    {
+      "epoch": 1.5670731707317072,
+      "grad_norm": 1.6599416732788086,
+      "learning_rate": 4.79449915567145e-06,
+      "loss": 0.1206,
+      "step": 172
+    },
+    {
+      "epoch": 1.576219512195122,
+      "grad_norm": 1.5868676900863647,
+      "learning_rate": 4.788833458198395e-06,
+      "loss": 0.1158,
+      "step": 173
+    },
+    {
+      "epoch": 1.5853658536585367,
+      "grad_norm": 1.0154211521148682,
+      "learning_rate": 4.783094158110175e-06,
+      "loss": 0.1091,
+      "step": 174
+    },
+    {
+      "epoch": 1.5945121951219512,
+      "grad_norm": 2.1514394283294678,
+      "learning_rate": 4.7772814399669585e-06,
+      "loss": 0.1109,
+      "step": 175
+    },
+    {
+      "epoch": 1.6036585365853657,
+      "grad_norm": 0.7475085258483887,
+      "learning_rate": 4.7713954906898415e-06,
+      "loss": 0.1162,
+      "step": 176
+    },
+    {
+      "epoch": 1.6128048780487805,
+      "grad_norm": 0.8523163795471191,
+      "learning_rate": 4.7654364995548325e-06,
+      "loss": 0.1238,
+      "step": 177
+    },
+    {
+      "epoch": 1.6219512195121952,
+      "grad_norm": 1.5456233024597168,
+      "learning_rate": 4.759404658186765e-06,
+      "loss": 0.1163,
+      "step": 178
+    },
+    {
+      "epoch": 1.6310975609756098,
+      "grad_norm": 1.123340129852295,
+      "learning_rate": 4.753300160553137e-06,
+      "loss": 0.1104,
+      "step": 179
+    },
+    {
+      "epoch": 1.6402439024390243,
+      "grad_norm": 1.16841721534729,
+      "learning_rate": 4.747123202957871e-06,
+      "loss": 0.1099,
+      "step": 180
+    },
+    {
+      "epoch": 1.649390243902439,
+      "grad_norm": 2.025221109390259,
+      "learning_rate": 4.740873984035009e-06,
+      "loss": 0.1157,
+      "step": 181
+    },
+    {
+      "epoch": 1.6585365853658538,
+      "grad_norm": 3.876469373703003,
+      "learning_rate": 4.734552704742314e-06,
+      "loss": 0.1122,
+      "step": 182
+    },
+    {
+      "epoch": 1.6676829268292683,
+      "grad_norm": 1.5562145709991455,
+      "learning_rate": 4.728159568354814e-06,
+      "loss": 0.1127,
+      "step": 183
+    },
+    {
+      "epoch": 1.6768292682926829,
+      "grad_norm": 1.4691755771636963,
+      "learning_rate": 4.721694780458266e-06,
+      "loss": 0.1182,
+      "step": 184
+    },
+    {
+      "epoch": 1.6859756097560976,
+      "grad_norm": 1.9529973268508911,
+      "learning_rate": 4.715158548942541e-06,
+      "loss": 0.1153,
+      "step": 185
+    },
+    {
+      "epoch": 1.6951219512195121,
+      "grad_norm": 1.9065771102905273,
+      "learning_rate": 4.708551083994944e-06,
+      "loss": 0.1163,
+      "step": 186
+    },
+    {
+      "epoch": 1.7042682926829267,
+      "grad_norm": 1.0964735746383667,
+      "learning_rate": 4.701872598093452e-06,
+      "loss": 0.114,
+      "step": 187
+    },
+    {
+      "epoch": 1.7134146341463414,
+      "grad_norm": 1.0885926485061646,
+      "learning_rate": 4.695123305999877e-06,
+      "loss": 0.114,
+      "step": 188
+    },
+    {
+      "epoch": 1.7225609756097562,
+      "grad_norm": 1.1777565479278564,
+      "learning_rate": 4.688303424752969e-06,
+      "loss": 0.1116,
+      "step": 189
+    },
+    {
+      "epoch": 1.7317073170731707,
+      "grad_norm": 1.3686976432800293,
+      "learning_rate": 4.6814131736614286e-06,
+      "loss": 0.1081,
+      "step": 190
+    },
+    {
+      "epoch": 1.7408536585365852,
+      "grad_norm": 0.8220391869544983,
+      "learning_rate": 4.674452774296861e-06,
+      "loss": 0.1091,
+      "step": 191
+    },
+    {
+      "epoch": 1.75,
+      "grad_norm": 1.073935866355896,
+      "learning_rate": 4.667422450486646e-06,
+      "loss": 0.1148,
+      "step": 192
+    },
+    {
+      "epoch": 1.7591463414634148,
+      "grad_norm": 1.389614224433899,
+      "learning_rate": 4.660322428306741e-06,
+      "loss": 0.1134,
+      "step": 193
+    },
+    {
+      "epoch": 1.7682926829268293,
+      "grad_norm": 1.349023699760437,
+      "learning_rate": 4.653152936074413e-06,
+      "loss": 0.1056,
+      "step": 194
+    },
+    {
+      "epoch": 1.7774390243902438,
+      "grad_norm": 1.0334906578063965,
+      "learning_rate": 4.645914204340898e-06,
+      "loss": 0.113,
+      "step": 195
+    },
+    {
+      "epoch": 1.7865853658536586,
+      "grad_norm": 18.6312313079834,
+      "learning_rate": 4.638606465883979e-06,
+      "loss": 0.1351,
+      "step": 196
+    },
+    {
+      "epoch": 1.7957317073170733,
+      "grad_norm": 0.7669345736503601,
+      "learning_rate": 4.6312299557005125e-06,
+      "loss": 0.1097,
+      "step": 197
+    },
+    {
+      "epoch": 1.8048780487804879,
+      "grad_norm": 2.1091127395629883,
+      "learning_rate": 4.623784910998862e-06,
+      "loss": 0.1122,
+      "step": 198
+    },
+    {
+      "epoch": 1.8140243902439024,
+      "grad_norm": 0.8031914830207825,
+      "learning_rate": 4.616271571191273e-06,
+      "loss": 0.1109,
+      "step": 199
+    },
+    {
+      "epoch": 1.8231707317073171,
+      "grad_norm": 2.1154661178588867,
+      "learning_rate": 4.608690177886177e-06,
+      "loss": 0.1131,
+      "step": 200
+    },
+    {
+      "epoch": 1.8323170731707317,
+      "grad_norm": 1.4946709871292114,
+      "learning_rate": 4.6010409748804165e-06,
+      "loss": 0.1136,
+      "step": 201
+    },
+    {
+      "epoch": 1.8414634146341462,
+      "grad_norm": 1.0509642362594604,
+      "learning_rate": 4.593324208151412e-06,
+      "loss": 0.1162,
+      "step": 202
+    },
+    {
+      "epoch": 1.850609756097561,
+      "grad_norm": 0.8742370009422302,
+      "learning_rate": 4.585540125849244e-06,
+      "loss": 0.1102,
+      "step": 203
+    },
+    {
+      "epoch": 1.8597560975609757,
+      "grad_norm": 1.1283056735992432,
+      "learning_rate": 4.577688978288681e-06,
+      "loss": 0.1104,
+      "step": 204
+    },
+    {
+      "epoch": 1.8689024390243902,
+      "grad_norm": 1.0081048011779785,
+      "learning_rate": 4.569771017941125e-06,
+      "loss": 0.1119,
+      "step": 205
+    },
+    {
+      "epoch": 1.8780487804878048,
+      "grad_norm": 0.7291656732559204,
+      "learning_rate": 4.561786499426494e-06,
+      "loss": 0.1103,
+      "step": 206
+    },
+    {
+      "epoch": 1.8871951219512195,
+      "grad_norm": 1.144096851348877,
+      "learning_rate": 4.5537356795050346e-06,
+      "loss": 0.1125,
+      "step": 207
+    },
+    {
+      "epoch": 1.8963414634146343,
+      "grad_norm": 0.7317456603050232,
+      "learning_rate": 4.545618817069066e-06,
+      "loss": 0.1105,
+      "step": 208
+    },
+    {
+      "epoch": 1.9054878048780488,
+      "grad_norm": 1.1636486053466797,
+      "learning_rate": 4.537436173134653e-06,
+      "loss": 0.1091,
+      "step": 209
+    },
+    {
+      "epoch": 1.9146341463414633,
+      "grad_norm": 0.6319037675857544,
+      "learning_rate": 4.529188010833212e-06,
+      "loss": 0.1088,
+      "step": 210
+    },
+    {
+      "epoch": 1.923780487804878,
+      "grad_norm": 0.7535589933395386,
+      "learning_rate": 4.520874595403053e-06,
+      "loss": 0.1041,
+      "step": 211
+    },
+    {
+      "epoch": 1.9329268292682928,
+      "grad_norm": 1.0461041927337646,
+      "learning_rate": 4.512496194180846e-06,
+      "loss": 0.1129,
+      "step": 212
+    },
+    {
+      "epoch": 1.9420731707317072,
+      "grad_norm": 0.6124874949455261,
+      "learning_rate": 4.5040530765930275e-06,
+      "loss": 0.1113,
+      "step": 213
+    },
+    {
+      "epoch": 1.951219512195122,
+      "grad_norm": 0.8516119122505188,
+      "learning_rate": 4.495545514147134e-06,
+      "loss": 0.1112,
+      "step": 214
+    },
+    {
+      "epoch": 1.9603658536585367,
+      "grad_norm": 0.8919075131416321,
+      "learning_rate": 4.486973780423073e-06,
+      "loss": 0.1089,
+      "step": 215
+    },
+    {
+      "epoch": 1.9695121951219512,
+      "grad_norm": 1.2183221578598022,
+      "learning_rate": 4.478338151064323e-06,
+      "loss": 0.1156,
+      "step": 216
+    },
+    {
+      "epoch": 1.9786585365853657,
+      "grad_norm": 1.0123646259307861,
+      "learning_rate": 4.4696389037690734e-06,
+      "loss": 0.1077,
+      "step": 217
+    },
+    {
+      "epoch": 1.9878048780487805,
+      "grad_norm": 0.8304896950721741,
+      "learning_rate": 4.460876318281291e-06,
+      "loss": 0.1051,
+      "step": 218
+    },
+    {
+      "epoch": 1.9969512195121952,
+      "grad_norm": 0.5879438519477844,
+      "learning_rate": 4.452050676381725e-06,
+      "loss": 0.1064,
+      "step": 219
+    },
+    {
+      "epoch": 2.0,
+      "grad_norm": 0.5879438519477844,
+      "learning_rate": 4.443162261878846e-06,
+      "loss": 0.0344,
+      "step": 220
+    },
+    {
+      "epoch": 2.0091463414634148,
+      "grad_norm": 0.8598087430000305,
+      "learning_rate": 4.434211360599721e-06,
+      "loss": 0.1071,
+      "step": 221
+    },
+    {
+      "epoch": 2.018292682926829,
+      "grad_norm": 0.5539699196815491,
+      "learning_rate": 4.425198260380818e-06,
+      "loss": 0.1095,
+      "step": 222
+    },
+    {
+      "epoch": 2.027439024390244,
+      "grad_norm": 0.4581831395626068,
+      "learning_rate": 4.416123251058756e-06,
+      "loss": 0.104,
+      "step": 223
+    },
+    {
+      "epoch": 2.0365853658536586,
+      "grad_norm": 0.550529420375824,
+      "learning_rate": 4.406986624460979e-06,
+      "loss": 0.1049,
+      "step": 224
+    },
+    {
+      "epoch": 2.0457317073170733,
+      "grad_norm": 0.749291181564331,
+      "learning_rate": 4.397788674396374e-06,
+      "loss": 0.1091,
+      "step": 225
+    },
+    {
+      "epoch": 2.0548780487804876,
+      "grad_norm": 0.7612536549568176,
+      "learning_rate": 4.3885296966458225e-06,
+      "loss": 0.1045,
+      "step": 226
+    },
+    {
+      "epoch": 2.0640243902439024,
+      "grad_norm": 0.6671517491340637,
+      "learning_rate": 4.37920998895269e-06,
+      "loss": 0.1057,
+      "step": 227
+    },
+    {
+      "epoch": 2.073170731707317,
+      "grad_norm": 0.5838347673416138,
+      "learning_rate": 4.369829851013251e-06,
+      "loss": 0.1061,
+      "step": 228
+    },
+    {
+      "epoch": 2.082317073170732,
+      "grad_norm": 1.937151312828064,
+      "learning_rate": 4.36038958446705e-06,
+      "loss": 0.1074,
+      "step": 229
+    },
+    {
+      "epoch": 2.091463414634146,
+      "grad_norm": 0.8083934187889099,
+      "learning_rate": 4.350889492887202e-06,
+      "loss": 0.1096,
+      "step": 230
+    },
+    {
+      "epoch": 2.100609756097561,
+      "grad_norm": 0.6810253262519836,
+      "learning_rate": 4.341329881770635e-06,
+      "loss": 0.1024,
+      "step": 231
+    },
+    {
+      "epoch": 2.1097560975609757,
+      "grad_norm": 0.8901084065437317,
+      "learning_rate": 4.331711058528259e-06,
+      "loss": 0.1064,
+      "step": 232
+    },
+    {
+      "epoch": 2.1189024390243905,
+      "grad_norm": 0.34419459104537964,
+      "learning_rate": 4.322033332475084e-06,
+      "loss": 0.1074,
+      "step": 233
+    },
+    {
+      "epoch": 2.1280487804878048,
+      "grad_norm": 0.8596321940422058,
+      "learning_rate": 4.312297014820275e-06,
+      "loss": 0.1061,
+      "step": 234
+    },
+    {
+      "epoch": 2.1371951219512195,
+      "grad_norm": 0.8391798138618469,
+      "learning_rate": 4.302502418657138e-06,
+      "loss": 0.107,
+      "step": 235
+    },
+    {
+      "epoch": 2.1463414634146343,
+      "grad_norm": 0.7193841338157654,
+      "learning_rate": 4.292649858953063e-06,
+      "loss": 0.1083,
+      "step": 236
+    },
+    {
+      "epoch": 2.1554878048780486,
+      "grad_norm": 1.5975165367126465,
+      "learning_rate": 4.282739652539384e-06,
+      "loss": 0.109,
+      "step": 237
+    },
+    {
+      "epoch": 2.1646341463414633,
+      "grad_norm": 0.9933533072471619,
+      "learning_rate": 4.272772118101195e-06,
+      "loss": 0.1073,
+      "step": 238
+    },
+    {
+      "epoch": 2.173780487804878,
+      "grad_norm": 0.9281901717185974,
+      "learning_rate": 4.262747576167106e-06,
+      "loss": 0.1066,
+      "step": 239
+    },
+    {
+      "epoch": 2.182926829268293,
+      "grad_norm": 3.2005884647369385,
+      "learning_rate": 4.2526663490989264e-06,
+      "loss": 0.1106,
+      "step": 240
+    },
+    {
+      "epoch": 2.192073170731707,
+      "grad_norm": 1.084905982017517,
+      "learning_rate": 4.24252876108131e-06,
+      "loss": 0.1047,
+      "step": 241
+    },
+    {
+      "epoch": 2.201219512195122,
+      "grad_norm": 0.5345124006271362,
+      "learning_rate": 4.232335138111321e-06,
+      "loss": 0.1027,
+      "step": 242
+    },
+    {
+      "epoch": 2.2103658536585367,
+      "grad_norm": 0.6235584020614624,
+      "learning_rate": 4.222085807987954e-06,
+      "loss": 0.1032,
+      "step": 243
+    },
+    {
+      "epoch": 2.2195121951219514,
+      "grad_norm": 0.9797384738922119,
+      "learning_rate": 4.211781100301596e-06,
+      "loss": 0.1082,
+      "step": 244
+    },
+    {
+      "epoch": 2.2286585365853657,
+      "grad_norm": 0.3665165901184082,
+      "learning_rate": 4.201421346423421e-06,
+      "loss": 0.1057,
+      "step": 245
+    },
+    {
+      "epoch": 2.2378048780487805,
+      "grad_norm": 1.232487440109253,
+      "learning_rate": 4.19100687949474e-06,
+      "loss": 0.1088,
+      "step": 246
+    },
+    {
+      "epoch": 2.2469512195121952,
+      "grad_norm": 0.8028481602668762,
+      "learning_rate": 4.180538034416287e-06,
+      "loss": 0.1119,
+      "step": 247
+    },
+    {
+      "epoch": 2.2560975609756095,
+      "grad_norm": 1.094025731086731,
+      "learning_rate": 4.1700151478374436e-06,
+      "loss": 0.1033,
+      "step": 248
+    },
+    {
+      "epoch": 2.2652439024390243,
+      "grad_norm": 1.0119608640670776,
+      "learning_rate": 4.159438558145425e-06,
+      "loss": 0.108,
+      "step": 249
+    },
+    {
+      "epoch": 2.274390243902439,
+      "grad_norm": 0.6864210963249207,
+      "learning_rate": 4.148808605454385e-06,
+      "loss": 0.1072,
+      "step": 250
+    },
+    {
+      "epoch": 2.283536585365854,
+      "grad_norm": 0.8415966629981995,
+      "learning_rate": 4.138125631594488e-06,
+      "loss": 0.1052,
+      "step": 251
+    },
+    {
+      "epoch": 2.292682926829268,
+      "grad_norm": 0.7150280475616455,
+      "learning_rate": 4.127389980100916e-06,
+      "loss": 0.1049,
+      "step": 252
+    },
+    {
+      "epoch": 2.301829268292683,
+      "grad_norm": 1.2521971464157104,
+      "learning_rate": 4.116601996202815e-06,
+      "loss": 0.1078,
+      "step": 253
+    },
+    {
+      "epoch": 2.3109756097560976,
+      "grad_norm": 0.8204450011253357,
+      "learning_rate": 4.1057620268122e-06,
+      "loss": 0.1078,
+      "step": 254
+    },
+    {
+      "epoch": 2.3201219512195124,
+      "grad_norm": 1.0992320775985718,
+      "learning_rate": 4.0948704205128e-06,
+      "loss": 0.1063,
+      "step": 255
+    },
+    {
+      "epoch": 2.3292682926829267,
+      "grad_norm": 0.4928065240383148,
+      "learning_rate": 4.08392752754884e-06,
+      "loss": 0.1076,
+      "step": 256
+    },
+    {
+      "epoch": 2.3384146341463414,
+      "grad_norm": 1.1588791608810425,
+      "learning_rate": 4.072933699813788e-06,
+      "loss": 0.1075,
+      "step": 257
+    },
+    {
+      "epoch": 2.347560975609756,
+      "grad_norm": 0.5810270309448242,
+      "learning_rate": 4.061889290839032e-06,
+      "loss": 0.107,
+      "step": 258
+    },
+    {
+      "epoch": 2.3567073170731705,
+      "grad_norm": 0.8428746461868286,
+      "learning_rate": 4.050794655782515e-06,
+      "loss": 0.1057,
+      "step": 259
+    },
+    {
+      "epoch": 2.3658536585365852,
+      "grad_norm": 1.4898464679718018,
+      "learning_rate": 4.039650151417316e-06,
+      "loss": 0.1043,
+      "step": 260
+    },
+    {
+      "epoch": 2.375,
+      "grad_norm": 0.8613987565040588,
+      "learning_rate": 4.028456136120169e-06,
+      "loss": 0.1011,
+      "step": 261
+    },
+    {
+      "epoch": 2.3841463414634148,
+      "grad_norm": 0.7771743535995483,
+      "learning_rate": 4.017212969859948e-06,
+      "loss": 0.1073,
+      "step": 262
+    },
+    {
+      "epoch": 2.3932926829268295,
+      "grad_norm": 0.4884893596172333,
+      "learning_rate": 4.0059210141860875e-06,
+      "loss": 0.104,
+      "step": 263
+    },
+    {
+      "epoch": 2.402439024390244,
+      "grad_norm": 0.8188875317573547,
+      "learning_rate": 3.994580632216952e-06,
+      "loss": 0.1043,
+      "step": 264
+    },
+    {
+      "epoch": 2.4115853658536586,
+      "grad_norm": 0.814239501953125,
+      "learning_rate": 3.9831921886281675e-06,
+      "loss": 0.1073,
+      "step": 265
+    },
+    {
+      "epoch": 2.4207317073170733,
+      "grad_norm": 1.029391884803772,
+      "learning_rate": 3.971756049640888e-06,
+      "loss": 0.103,
+      "step": 266
+    },
+    {
+      "epoch": 2.4298780487804876,
+      "grad_norm": 0.3826993405818939,
+      "learning_rate": 3.96027258301002e-06,
+      "loss": 0.1089,
+      "step": 267
+    },
+    {
+      "epoch": 2.4390243902439024,
+      "grad_norm": 0.5306946039199829,
+      "learning_rate": 3.9487421580123995e-06,
+      "loss": 0.1086,
+      "step": 268
+    },
+    {
+      "epoch": 2.448170731707317,
+      "grad_norm": 1.286549687385559,
+      "learning_rate": 3.937165145434914e-06,
+      "loss": 0.1085,
+      "step": 269
+    },
+    {
+      "epoch": 2.457317073170732,
+      "grad_norm": 1.5009950399398804,
+      "learning_rate": 3.92554191756258e-06,
+      "loss": 0.1068,
+      "step": 270
+    },
+    {
+      "epoch": 2.466463414634146,
+      "grad_norm": 0.9225317239761353,
+      "learning_rate": 3.9138728481665696e-06,
+      "loss": 0.1059,
+      "step": 271
+    },
+    {
+      "epoch": 2.475609756097561,
+      "grad_norm": 0.5062177181243896,
+      "learning_rate": 3.902158312492196e-06,
+      "loss": 0.1089,
+      "step": 272
+    },
+    {
+      "epoch": 2.4847560975609757,
+      "grad_norm": 0.6272971630096436,
+      "learning_rate": 3.8903986872468416e-06,
+      "loss": 0.1099,
+      "step": 273
+    },
+    {
+      "epoch": 2.4939024390243905,
+      "grad_norm": 1.207027554512024,
+      "learning_rate": 3.878594350587844e-06,
+      "loss": 0.1086,
+      "step": 274
+    },
+    {
+      "epoch": 2.5030487804878048,
+      "grad_norm": 0.8417999744415283,
+      "learning_rate": 3.866745682110343e-06,
+      "loss": 0.1091,
+      "step": 275
+    },
+    {
+      "epoch": 2.5121951219512195,
+      "grad_norm": 0.5926170945167542,
+      "learning_rate": 3.854853062835062e-06,
+      "loss": 0.1081,
+      "step": 276
+    },
+    {
+      "epoch": 2.5213414634146343,
+      "grad_norm": 0.5806312561035156,
+      "learning_rate": 3.842916875196066e-06,
+      "loss": 0.1057,
+      "step": 277
+    },
+    {
+      "epoch": 2.5304878048780486,
+      "grad_norm": 1.2453746795654297,
+      "learning_rate": 3.830937503028457e-06,
+      "loss": 0.1086,
+      "step": 278
+    },
+    {
+      "epoch": 2.5396341463414633,
+      "grad_norm": 0.32152408361434937,
+      "learning_rate": 3.818915331556032e-06,
+      "loss": 0.1076,
+      "step": 279
+    },
+    {
+      "epoch": 2.548780487804878,
+      "grad_norm": 0.5756719708442688,
+      "learning_rate": 3.8068507473789016e-06,
+      "loss": 0.1057,
+      "step": 280
+    },
+    {
+      "epoch": 2.557926829268293,
+      "grad_norm": 0.9345139861106873,
+      "learning_rate": 3.7947441384610473e-06,
+      "loss": 0.1043,
+      "step": 281
+    },
+    {
+      "epoch": 2.567073170731707,
+      "grad_norm": 0.6475419998168945,
+      "learning_rate": 3.7825958941178538e-06,
+      "loss": 0.1046,
+      "step": 282
+    },
+    {
+      "epoch": 2.576219512195122,
+      "grad_norm": 0.7345425486564636,
+      "learning_rate": 3.770406405003586e-06,
+      "loss": 0.1042,
+      "step": 283
+    },
+    {
+      "epoch": 2.5853658536585367,
+      "grad_norm": 0.769500195980072,
+      "learning_rate": 3.758176063098829e-06,
+      "loss": 0.1048,
+      "step": 284
+    },
+    {
+      "epoch": 2.5945121951219514,
+      "grad_norm": 0.5934444665908813,
+      "learning_rate": 3.745905261697881e-06,
+      "loss": 0.1086,
+      "step": 285
+    },
+    {
+      "epoch": 2.6036585365853657,
+      "grad_norm": 0.5718421936035156,
+      "learning_rate": 3.7335943953961063e-06,
+      "loss": 0.1027,
+      "step": 286
+    },
+    {
+      "epoch": 2.6128048780487805,
+      "grad_norm": 0.6114512085914612,
+      "learning_rate": 3.721243860077247e-06,
+      "loss": 0.1091,
+      "step": 287
+    },
+    {
+      "epoch": 2.6219512195121952,
+      "grad_norm": 0.5620512366294861,
+      "learning_rate": 3.708854052900693e-06,
+      "loss": 0.1088,
+      "step": 288
+    },
+    {
+      "epoch": 2.6310975609756095,
+      "grad_norm": 0.6152999997138977,
+      "learning_rate": 3.696425372288708e-06,
+      "loss": 0.1035,
+      "step": 289
+    },
+    {
+      "epoch": 2.6402439024390243,
+      "grad_norm": 0.9731057286262512,
+      "learning_rate": 3.68395821791362e-06,
+      "loss": 0.1066,
+      "step": 290
+    },
+    {
+      "epoch": 2.649390243902439,
+      "grad_norm": 0.49733713269233704,
+      "learning_rate": 3.6714529906849683e-06,
+      "loss": 0.1045,
+      "step": 291
+    },
+    {
+      "epoch": 2.658536585365854,
+      "grad_norm": 0.49706941843032837,
+      "learning_rate": 3.65891009273661e-06,
+      "loss": 0.1035,
+      "step": 292
+    },
+    {
+      "epoch": 2.6676829268292686,
+      "grad_norm": 0.32593467831611633,
+      "learning_rate": 3.6463299274137883e-06,
+      "loss": 0.1017,
+      "step": 293
+    },
+    {
+      "epoch": 2.676829268292683,
+      "grad_norm": 0.4569077491760254,
+      "learning_rate": 3.6337128992601665e-06,
+      "loss": 0.1061,
+      "step": 294
+    },
+    {
+      "epoch": 2.6859756097560976,
+      "grad_norm": 0.5540208220481873,
+      "learning_rate": 3.621059414004811e-06,
+      "loss": 0.106,
+      "step": 295
+    },
+    {
+      "epoch": 2.6951219512195124,
+      "grad_norm": 0.9319402575492859,
+      "learning_rate": 3.6083698785491537e-06,
+      "loss": 0.1042,
+      "step": 296
+    },
+    {
+      "epoch": 2.7042682926829267,
+      "grad_norm": 0.795711874961853,
+      "learning_rate": 3.595644700953898e-06,
+      "loss": 0.104,
+      "step": 297
+    },
+    {
+      "epoch": 2.7134146341463414,
+      "grad_norm": 0.4707932472229004,
+      "learning_rate": 3.582884290425903e-06,
+      "loss": 0.1087,
+      "step": 298
+    },
+    {
+      "epoch": 2.722560975609756,
+      "grad_norm": 0.7709797620773315,
+      "learning_rate": 3.5700890573050224e-06,
+      "loss": 0.107,
+      "step": 299
+    },
+    {
+      "epoch": 2.7317073170731705,
+      "grad_norm": 1.1342726945877075,
+      "learning_rate": 3.5572594130509074e-06,
+      "loss": 0.1029,
+      "step": 300
+    },
+    {
+      "epoch": 2.7408536585365852,
+      "grad_norm": 0.9055543541908264,
+      "learning_rate": 3.544395770229781e-06,
+      "loss": 0.1087,
+      "step": 301
+    },
+    {
+      "epoch": 2.75,
+      "grad_norm": 1.084656000137329,
+      "learning_rate": 3.531498542501161e-06,
+      "loss": 0.1044,
+      "step": 302
+    },
+    {
+      "epoch": 2.7591463414634148,
+      "grad_norm": 0.5991970300674438,
+      "learning_rate": 3.51856814460457e-06,
+      "loss": 0.1064,
+      "step": 303
+    },
+    {
+      "epoch": 2.7682926829268295,
+      "grad_norm": 1.7364325523376465,
+      "learning_rate": 3.5056049923461873e-06,
+      "loss": 0.1079,
+      "step": 304
+    },
+    {
+      "epoch": 2.777439024390244,
+      "grad_norm": 0.4037036895751953,
+      "learning_rate": 3.4926095025854863e-06,
+      "loss": 0.1058,
+      "step": 305
+    },
+    {
+      "epoch": 2.7865853658536586,
+      "grad_norm": 0.8944678902626038,
+      "learning_rate": 3.479582093221824e-06,
+      "loss": 0.1064,
+      "step": 306
+    },
+    {
+      "epoch": 2.7957317073170733,
+      "grad_norm": 0.5730510950088501,
+      "learning_rate": 3.4665231831810055e-06,
+      "loss": 0.1055,
+      "step": 307
+    },
+    {
+      "epoch": 2.8048780487804876,
+      "grad_norm": 0.519359827041626,
+      "learning_rate": 3.45343319240181e-06,
+      "loss": 0.1069,
+      "step": 308
+    },
+    {
+      "epoch": 2.8140243902439024,
+      "grad_norm": 0.6074658632278442,
+      "learning_rate": 3.44031254182249e-06,
+      "loss": 0.1058,
+      "step": 309
+    },
+    {
+      "epoch": 2.823170731707317,
+      "grad_norm": 0.37584108114242554,
+      "learning_rate": 3.4271616533672312e-06,
+      "loss": 0.1052,
+      "step": 310
+    },
+    {
+      "epoch": 2.8323170731707314,
+      "grad_norm": 0.7730855941772461,
+      "learning_rate": 3.413980949932589e-06,
+      "loss": 0.1037,
+      "step": 311
+    },
+    {
+      "epoch": 2.841463414634146,
+      "grad_norm": 0.9912458062171936,
+      "learning_rate": 3.4007708553738837e-06,
+      "loss": 0.1039,
+      "step": 312
+    },
+    {
+      "epoch": 2.850609756097561,
+      "grad_norm": 0.36861947178840637,
+      "learning_rate": 3.3875317944915765e-06,
+      "loss": 0.1083,
+      "step": 313
+    },
+    {
+      "epoch": 2.8597560975609757,
+      "grad_norm": 0.6069959402084351,
+      "learning_rate": 3.3742641930176044e-06,
+      "loss": 0.1063,
+      "step": 314
+    },
+    {
+      "epoch": 2.8689024390243905,
+      "grad_norm": 0.5478480458259583,
+      "learning_rate": 3.3609684776016932e-06,
+      "loss": 0.1033,
+      "step": 315
+    },
+    {
+      "epoch": 2.8780487804878048,
+      "grad_norm": 0.8352198004722595,
+      "learning_rate": 3.3476450757976353e-06,
+      "loss": 0.1026,
+      "step": 316
+    },
+    {
+      "epoch": 2.8871951219512195,
+      "grad_norm": 0.32641175389289856,
+      "learning_rate": 3.3342944160495405e-06,
+      "loss": 0.1065,
+      "step": 317
+    },
+    {
+      "epoch": 2.8963414634146343,
+      "grad_norm": 0.8076167702674866,
+      "learning_rate": 3.3209169276780612e-06,
+      "loss": 0.105,
+      "step": 318
+    },
+    {
+      "epoch": 2.9054878048780486,
+      "grad_norm": 0.3602111041545868,
+      "learning_rate": 3.307513040866584e-06,
+      "loss": 0.106,
+      "step": 319
+    },
+    {
+      "epoch": 2.9146341463414633,
+      "grad_norm": 0.227371945977211,
+      "learning_rate": 3.2940831866473967e-06,
+      "loss": 0.1029,
+      "step": 320
+    },
+    {
+      "epoch": 2.923780487804878,
+      "grad_norm": 0.5158879160881042,
+      "learning_rate": 3.280627796887828e-06,
+      "loss": 0.1046,
+      "step": 321
+    },
+    {
+      "epoch": 2.932926829268293,
+      "grad_norm": 1.0091784000396729,
+      "learning_rate": 3.2671473042763603e-06,
+      "loss": 0.1063,
+      "step": 322
+    },
+    {
+      "epoch": 2.942073170731707,
+      "grad_norm": 0.9310367107391357,
+      "learning_rate": 3.2536421423087137e-06,
+      "loss": 0.1015,
+      "step": 323
+    },
+    {
+      "epoch": 2.951219512195122,
+      "grad_norm": 0.7602329254150391,
+      "learning_rate": 3.2401127452739106e-06,
+      "loss": 0.1061,
+      "step": 324
+    },
+    {
+      "epoch": 2.9603658536585367,
+      "grad_norm": 0.3374696671962738,
+      "learning_rate": 3.2265595482403002e-06,
+      "loss": 0.1049,
+      "step": 325
+    },
+    {
+      "epoch": 2.9695121951219514,
+      "grad_norm": 0.8469128012657166,
+      "learning_rate": 3.212982987041582e-06,
+      "loss": 0.1059,
+      "step": 326
+    },
+    {
+      "epoch": 2.9786585365853657,
+      "grad_norm": 0.7546894550323486,
+      "learning_rate": 3.199383498262777e-06,
+      "loss": 0.1067,
+      "step": 327
+    },
+    {
+      "epoch": 2.9878048780487805,
+      "grad_norm": 0.471538245677948,
+      "learning_rate": 3.185761519226199e-06,
+      "loss": 0.1054,
+      "step": 328
+    },
+    {
+      "epoch": 2.9969512195121952,
+      "grad_norm": 1.0999424457550049,
+      "learning_rate": 3.1721174879773825e-06,
+      "loss": 0.1067,
+      "step": 329
+    },
+    {
+      "epoch": 3.0,
+      "grad_norm": 0.9866787791252136,
+      "learning_rate": 3.1584518432710055e-06,
+      "loss": 0.0361,
+      "step": 330
+    },
+    {
+      "epoch": 3.0091463414634148,
+      "grad_norm": 0.4876601994037628,
+      "learning_rate": 3.1447650245567714e-06,
+      "loss": 0.1058,
+      "step": 331
+    },
+    {
+      "epoch": 3.018292682926829,
+      "grad_norm": 1.1444916725158691,
+      "learning_rate": 3.131057471965283e-06,
+      "loss": 0.1051,
+      "step": 332
+    },
+    {
+      "epoch": 3.027439024390244,
+      "grad_norm": 0.6264286637306213,
+      "learning_rate": 3.117329626293891e-06,
+      "loss": 0.1065,
+      "step": 333
+    },
+    {
+      "epoch": 3.0365853658536586,
+      "grad_norm": 0.49345099925994873,
+      "learning_rate": 3.10358192899251e-06,
+      "loss": 0.1034,
+      "step": 334
+    },
+    {
+      "epoch": 3.0457317073170733,
+      "grad_norm": 0.663642168045044,
+      "learning_rate": 3.089814822149435e-06,
+      "loss": 0.1063,
+      "step": 335
+    },
+    {
+      "epoch": 3.0548780487804876,
+      "grad_norm": 0.7982928156852722,
+      "learning_rate": 3.0760287484771124e-06,
+      "loss": 0.1047,
+      "step": 336
+    },
+    {
+      "epoch": 3.0640243902439024,
+      "grad_norm": 0.7954766750335693,
+      "learning_rate": 3.062224151297915e-06,
+      "loss": 0.1069,
+      "step": 337
+    },
+    {
+      "epoch": 3.073170731707317,
+      "grad_norm": 0.42658311128616333,
+      "learning_rate": 3.0484014745298784e-06,
+      "loss": 0.1043,
+      "step": 338
+    },
+    {
+      "epoch": 3.082317073170732,
+      "grad_norm": 0.8154664635658264,
+      "learning_rate": 3.0345611626724286e-06,
+      "loss": 0.1061,
+      "step": 339
+    },
+    {
+      "epoch": 3.091463414634146,
+      "grad_norm": 0.6405104994773865,
+      "learning_rate": 3.020703660792089e-06,
+      "loss": 0.1044,
+      "step": 340
+    },
+    {
+      "epoch": 3.100609756097561,
+      "grad_norm": 0.7365081906318665,
+      "learning_rate": 3.006829414508164e-06,
+      "loss": 0.1061,
+      "step": 341
+    },
+    {
+      "epoch": 3.1097560975609757,
+      "grad_norm": 0.322336345911026,
+      "learning_rate": 2.9929388699784163e-06,
+      "loss": 0.1067,
+      "step": 342
+    },
+    {
+      "epoch": 3.1189024390243905,
+      "grad_norm": 1.0929139852523804,
+      "learning_rate": 2.9790324738847117e-06,
+      "loss": 0.1084,
+      "step": 343
+    },
+    {
+      "epoch": 3.1280487804878048,
+      "grad_norm": 0.9065106511116028,
+      "learning_rate": 2.965110673418662e-06,
+      "loss": 0.1029,
+      "step": 344
+    },
+    {
+      "epoch": 3.1371951219512195,
+      "grad_norm": 0.7359282970428467,
+      "learning_rate": 2.9511739162672377e-06,
+      "loss": 0.1022,
+      "step": 345
+    },
+    {
+      "epoch": 3.1463414634146343,
+      "grad_norm": 0.5595942139625549,
+      "learning_rate": 2.9372226505983803e-06,
+      "loss": 0.1067,
+      "step": 346
+    },
+    {
+      "epoch": 3.1554878048780486,
+      "grad_norm": 0.9696919918060303,
+      "learning_rate": 2.923257325046581e-06,
+      "loss": 0.1085,
+      "step": 347
+    },
+    {
+      "epoch": 3.1646341463414633,
+      "grad_norm": 0.3462243676185608,
+      "learning_rate": 2.9092783886984615e-06,
+      "loss": 0.1022,
+      "step": 348
+    },
+    {
+      "epoch": 3.173780487804878,
+      "grad_norm": 0.5291770696640015,
+      "learning_rate": 2.895286291078327e-06,
+      "loss": 0.1053,
+      "step": 349
+    },
+    {
+      "epoch": 3.182926829268293,
+      "grad_norm": 0.667945921421051,
+      "learning_rate": 2.881281482133715e-06,
+      "loss": 0.1031,
+      "step": 350
+    },
+    {
+      "epoch": 3.192073170731707,
+      "grad_norm": 1.4323768615722656,
+      "learning_rate": 2.8672644122209224e-06,
+      "loss": 0.1081,
+      "step": 351
+    },
+    {
+      "epoch": 3.201219512195122,
+      "grad_norm": 0.6841598749160767,
+      "learning_rate": 2.8532355320905277e-06,
+      "loss": 0.106,
+      "step": 352
+    },
+    {
+      "epoch": 3.2103658536585367,
+      "grad_norm": 1.017216444015503,
+      "learning_rate": 2.839195292872892e-06,
+      "loss": 0.1087,
+      "step": 353
+    },
+    {
+      "epoch": 3.2195121951219514,
+      "grad_norm": 0.6531134843826294,
+      "learning_rate": 2.8251441460636542e-06,
+      "loss": 0.1046,
+      "step": 354
+    },
+    {
+      "epoch": 3.2286585365853657,
+      "grad_norm": 0.5456640124320984,
+      "learning_rate": 2.8110825435092108e-06,
+      "loss": 0.1039,
+      "step": 355
+    },
+    {
+      "epoch": 3.2378048780487805,
+      "grad_norm": 0.7548927068710327,
+      "learning_rate": 2.797010937392188e-06,
+      "loss": 0.1031,
+      "step": 356
+    },
+    {
+      "epoch": 3.2469512195121952,
+      "grad_norm": 0.3874664008617401,
+      "learning_rate": 2.782929780216896e-06,
+      "loss": 0.1044,
+      "step": 357
+    },
+    {
+      "epoch": 3.2560975609756095,
+      "grad_norm": 0.41182073950767517,
+      "learning_rate": 2.768839524794784e-06,
+      "loss": 0.1041,
+      "step": 358
+    },
+    {
+      "epoch": 3.2652439024390243,
+      "grad_norm": 0.9750642776489258,
+      "learning_rate": 2.754740624229875e-06,
+      "loss": 0.1074,
+      "step": 359
+    },
+    {
+      "epoch": 3.274390243902439,
+      "grad_norm": 0.5468454957008362,
+      "learning_rate": 2.740633531904196e-06,
+      "loss": 0.1056,
+      "step": 360
+    },
+    {
+      "epoch": 3.283536585365854,
+      "grad_norm": 0.6265747547149658,
+      "learning_rate": 2.7265187014631983e-06,
+      "loss": 0.0993,
+      "step": 361
+    },
+    {
+      "epoch": 3.292682926829268,
+      "grad_norm": 0.5447110533714294,
+      "learning_rate": 2.7123965868011693e-06,
+      "loss": 0.1023,
+      "step": 362
+    },
+    {
+      "epoch": 3.301829268292683,
+      "grad_norm": 0.5873153805732727,
+      "learning_rate": 2.698267642046639e-06,
+      "loss": 0.1043,
+      "step": 363
+    },
+    {
+      "epoch": 3.3109756097560976,
+      "grad_norm": 0.8410884141921997,
+      "learning_rate": 2.6841323215477716e-06,
+      "loss": 0.1026,
+      "step": 364
+    },
+    {
+      "epoch": 3.3201219512195124,
+      "grad_norm": 0.6379916667938232,
+      "learning_rate": 2.6699910798577612e-06,
+      "loss": 0.1043,
+      "step": 365
+    },
+    {
+      "epoch": 3.3292682926829267,
+      "grad_norm": 0.3435944616794586,
+      "learning_rate": 2.6558443717202077e-06,
+      "loss": 0.105,
+      "step": 366
+    },
+    {
+      "epoch": 3.3384146341463414,
+      "grad_norm": 1.0160895586013794,
+      "learning_rate": 2.6416926520545e-06,
+      "loss": 0.1029,
+      "step": 367
+    },
+    {
+      "epoch": 3.347560975609756,
+      "grad_norm": 0.6623978018760681,
+      "learning_rate": 2.6275363759411817e-06,
+      "loss": 0.1023,
+      "step": 368
+    },
+    {
+      "epoch": 3.3567073170731705,
+      "grad_norm": 0.42660218477249146,
+      "learning_rate": 2.61337599860732e-06,
+      "loss": 0.102,
+      "step": 369
+    },
+    {
+      "epoch": 3.3658536585365852,
+      "grad_norm": 0.992641270160675,
+      "learning_rate": 2.599211975411867e-06,
+      "loss": 0.1087,
+      "step": 370
+    },
+    {
+      "epoch": 3.375,
+      "grad_norm": 0.9266735911369324,
+      "learning_rate": 2.585044761831014e-06,
+      "loss": 0.102,
+      "step": 371
+    },
+    {
+      "epoch": 3.3841463414634148,
+      "grad_norm": 0.7853158712387085,
+      "learning_rate": 2.5708748134435503e-06,
+      "loss": 0.1062,
+      "step": 372
+    },
+    {
+      "epoch": 3.3932926829268295,
+      "grad_norm": 0.501575767993927,
+      "learning_rate": 2.556702585916202e-06,
+      "loss": 0.1034,
+      "step": 373
+    },
+    {
+      "epoch": 3.402439024390244,
+      "grad_norm": 0.4489349126815796,
+      "learning_rate": 2.5425285349889937e-06,
+      "loss": 0.1072,
+      "step": 374
+    },
+    {
+      "epoch": 3.4115853658536586,
+      "grad_norm": 0.6775074601173401,
+      "learning_rate": 2.52835311646058e-06,
+      "loss": 0.1069,
+      "step": 375
+    },
+    {
+      "epoch": 3.4207317073170733,
+      "grad_norm": 0.6220641732215881,
+      "learning_rate": 2.5141767861735976e-06,
+      "loss": 0.1057,
+      "step": 376
+    },
+    {
+      "epoch": 3.4298780487804876,
+      "grad_norm": 0.8748292326927185,
+      "learning_rate": 2.5e-06,
+      "loss": 0.1043,
+      "step": 377
+    },
+    {
+      "epoch": 3.4390243902439024,
+      "grad_norm": 0.9639992117881775,
+      "learning_rate": 2.485823213826403e-06,
+      "loss": 0.1086,
+      "step": 378
+    },
+    {
+      "epoch": 3.448170731707317,
+      "grad_norm": 0.7457119822502136,
+      "learning_rate": 2.47164688353942e-06,
+      "loss": 0.1066,
+      "step": 379
+    },
+    {
+      "epoch": 3.457317073170732,
+      "grad_norm": 0.9384257197380066,
+      "learning_rate": 2.457471465011007e-06,
+      "loss": 0.1045,
+      "step": 380
+    },
+    {
+      "epoch": 3.466463414634146,
+      "grad_norm": 1.3412197828292847,
+      "learning_rate": 2.4432974140837988e-06,
+      "loss": 0.1041,
+      "step": 381
+    },
+    {
+      "epoch": 3.475609756097561,
+      "grad_norm": 0.5068745017051697,
+      "learning_rate": 2.4291251865564505e-06,
+      "loss": 0.1032,
+      "step": 382
+    },
+    {
+      "epoch": 3.4847560975609757,
+      "grad_norm": 0.40219390392303467,
+      "learning_rate": 2.414955238168986e-06,
+      "loss": 0.1039,
+      "step": 383
+    },
+    {
+      "epoch": 3.4939024390243905,
+      "grad_norm": 1.0244395732879639,
+      "learning_rate": 2.4007880245881345e-06,
+      "loss": 0.1049,
+      "step": 384
+    },
+    {
+      "epoch": 3.5030487804878048,
+      "grad_norm": 0.7072446942329407,
+      "learning_rate": 2.386624001392681e-06,
+      "loss": 0.1062,
+      "step": 385
+    },
+    {
+      "epoch": 3.5121951219512195,
+      "grad_norm": 0.7629527449607849,
+      "learning_rate": 2.3724636240588196e-06,
+      "loss": 0.1034,
+      "step": 386
+    },
+    {
+      "epoch": 3.5213414634146343,
+      "grad_norm": 0.8184047937393188,
+      "learning_rate": 2.358307347945501e-06,
+      "loss": 0.1018,
+      "step": 387
+    },
+    {
+      "epoch": 3.5304878048780486,
+      "grad_norm": 0.353141188621521,
+      "learning_rate": 2.3441556282797935e-06,
+      "loss": 0.1077,
+      "step": 388
+    },
+    {
+      "epoch": 3.5396341463414633,
+      "grad_norm": 0.572818398475647,
+      "learning_rate": 2.33000892014224e-06,
+      "loss": 0.1047,
+      "step": 389
+    },
+    {
+      "epoch": 3.548780487804878,
+      "grad_norm": 0.7230311632156372,
+      "learning_rate": 2.315867678452229e-06,
+      "loss": 0.1025,
+      "step": 390
+    },
+    {
+      "epoch": 3.557926829268293,
+      "grad_norm": 1.1688281297683716,
+      "learning_rate": 2.3017323579533617e-06,
+      "loss": 0.1036,
+      "step": 391
+    },
+    {
+      "epoch": 3.567073170731707,
+      "grad_norm": 0.8227838277816772,
+      "learning_rate": 2.287603413198831e-06,
+      "loss": 0.104,
+      "step": 392
+    },
+    {
+      "epoch": 3.576219512195122,
+      "grad_norm": 0.45366042852401733,
+      "learning_rate": 2.2734812985368034e-06,
+      "loss": 0.1042,
+      "step": 393
+    },
+    {
+      "epoch": 3.5853658536585367,
+      "grad_norm": 0.31546878814697266,
+      "learning_rate": 2.2593664680958048e-06,
+      "loss": 0.0998,
+      "step": 394
+    },
+    {
+      "epoch": 3.5945121951219514,
+      "grad_norm": 0.3552347719669342,
+      "learning_rate": 2.2452593757701254e-06,
+      "loss": 0.1032,
+      "step": 395
+    },
+    {
+      "epoch": 3.6036585365853657,
+      "grad_norm": 0.38970789313316345,
+      "learning_rate": 2.231160475205216e-06,
+      "loss": 0.1014,
+      "step": 396
+    },
+    {
+      "epoch": 3.6128048780487805,
+      "grad_norm": 0.32332485914230347,
+      "learning_rate": 2.2170702197831052e-06,
+      "loss": 0.1042,
+      "step": 397
+    },
+    {
+      "epoch": 3.6219512195121952,
+      "grad_norm": 0.5800347924232483,
+      "learning_rate": 2.2029890626078133e-06,
+      "loss": 0.1025,
+      "step": 398
+    },
+    {
+      "epoch": 3.6310975609756095,
+      "grad_norm": 0.5717971920967102,
+      "learning_rate": 2.1889174564907897e-06,
+      "loss": 0.1022,
+      "step": 399
+    },
+    {
+      "epoch": 3.6402439024390243,
+      "grad_norm": 0.4129480719566345,
+      "learning_rate": 2.1748558539363458e-06,
+      "loss": 0.1043,
+      "step": 400
+    },
+    {
+      "epoch": 3.649390243902439,
+      "grad_norm": 0.8331505656242371,
+      "learning_rate": 2.1608047071271087e-06,
+      "loss": 0.103,
+      "step": 401
+    },
+    {
+      "epoch": 3.658536585365854,
+      "grad_norm": 0.7323583364486694,
+      "learning_rate": 2.1467644679094727e-06,
+      "loss": 0.1066,
+      "step": 402
+    },
+    {
+      "epoch": 3.6676829268292686,
+      "grad_norm": 0.828615665435791,
+      "learning_rate": 2.132735587779078e-06,
+      "loss": 0.1029,
+      "step": 403
+    },
+    {
+      "epoch": 3.676829268292683,
+      "grad_norm": 0.35115236043930054,
+      "learning_rate": 2.118718517866286e-06,
+      "loss": 0.1005,
+      "step": 404
+    },
+    {
+      "epoch": 3.6859756097560976,
+      "grad_norm": 0.45387038588523865,
+      "learning_rate": 2.1047137089216728e-06,
+      "loss": 0.1043,
+      "step": 405
+    },
+    {
+      "epoch": 3.6951219512195124,
+      "grad_norm": 0.6801460981369019,
+      "learning_rate": 2.09072161130154e-06,
+      "loss": 0.1013,
+      "step": 406
+    },
+    {
+      "epoch": 3.7042682926829267,
+      "grad_norm": 0.5863631367683411,
+      "learning_rate": 2.0767426749534194e-06,
+      "loss": 0.1008,
+      "step": 407
+    },
+    {
+      "epoch": 3.7134146341463414,
+      "grad_norm": 0.8629023432731628,
+      "learning_rate": 2.0627773494016205e-06,
+      "loss": 0.1058,
+      "step": 408
+    },
+    {
+      "epoch": 3.722560975609756,
+      "grad_norm": 0.929577112197876,
+      "learning_rate": 2.0488260837327627e-06,
+      "loss": 0.1043,
+      "step": 409
+    },
+    {
+      "epoch": 3.7317073170731705,
+      "grad_norm": 0.4356759488582611,
+      "learning_rate": 2.0348893265813397e-06,
+      "loss": 0.1029,
+      "step": 410
+    },
+    {
+      "epoch": 3.7408536585365852,
+      "grad_norm": 1.105850100517273,
+      "learning_rate": 2.0209675261152896e-06,
+      "loss": 0.104,
+      "step": 411
+    },
+    {
+      "epoch": 3.75,
+      "grad_norm": 0.5208716988563538,
+      "learning_rate": 2.0070611300215846e-06,
+      "loss": 0.103,
+      "step": 412
+    },
+    {
+      "epoch": 3.7591463414634148,
+      "grad_norm": 0.6509872674942017,
+      "learning_rate": 1.993170585491836e-06,
+      "loss": 0.1032,
+      "step": 413
+    },
+    {
+      "epoch": 3.7682926829268295,
+      "grad_norm": 0.48762744665145874,
+      "learning_rate": 1.9792963392079125e-06,
+      "loss": 0.1065,
+      "step": 414
+    },
+    {
+      "epoch": 3.777439024390244,
+      "grad_norm": 0.4907792806625366,
+      "learning_rate": 1.9654388373275722e-06,
+      "loss": 0.1018,
+      "step": 415
+    },
+    {
+      "epoch": 3.7865853658536586,
+      "grad_norm": 0.6827414035797119,
+      "learning_rate": 1.951598525470122e-06,
+      "loss": 0.103,
+      "step": 416
+    },
+    {
+      "epoch": 3.7957317073170733,
+      "grad_norm": 0.915044367313385,
+      "learning_rate": 1.937775848702086e-06,
+      "loss": 0.1033,
+      "step": 417
+    },
+    {
+      "epoch": 3.8048780487804876,
+      "grad_norm": 0.7613033056259155,
+      "learning_rate": 1.923971251522888e-06,
+      "loss": 0.1047,
+      "step": 418
+    },
+    {
+      "epoch": 3.8140243902439024,
+      "grad_norm": 0.5619191527366638,
+      "learning_rate": 1.9101851778505666e-06,
+      "loss": 0.1031,
+      "step": 419
+    },
+    {
+      "epoch": 3.823170731707317,
+      "grad_norm": 0.8900383710861206,
+      "learning_rate": 1.8964180710074905e-06,
+      "loss": 0.1059,
+      "step": 420
+    },
+    {
+      "epoch": 3.8323170731707314,
+      "grad_norm": 1.0572234392166138,
+      "learning_rate": 1.8826703737061097e-06,
+      "loss": 0.1062,
+      "step": 421
+    },
+    {
+      "epoch": 3.841463414634146,
+      "grad_norm": 0.379605233669281,
+      "learning_rate": 1.8689425280347168e-06,
+      "loss": 0.1016,
+      "step": 422
+    },
+    {
+      "epoch": 3.850609756097561,
+      "grad_norm": 0.5158970355987549,
+      "learning_rate": 1.8552349754432303e-06,
+      "loss": 0.1044,
+      "step": 423
+    },
+    {
+      "epoch": 3.8597560975609757,
+      "grad_norm": 0.5448965430259705,
+      "learning_rate": 1.8415481567289956e-06,
+      "loss": 0.1055,
+      "step": 424
+    },
+    {
+      "epoch": 3.8689024390243905,
+      "grad_norm": 0.6450077891349792,
+      "learning_rate": 1.8278825120226179e-06,
+      "loss": 0.1015,
+      "step": 425
+    },
+    {
+      "epoch": 3.8780487804878048,
+      "grad_norm": 0.7701898813247681,
+      "learning_rate": 1.8142384807738023e-06,
+      "loss": 0.1019,
+      "step": 426
+    },
+    {
+      "epoch": 3.8871951219512195,
+      "grad_norm": 1.0020835399627686,
+      "learning_rate": 1.800616501737224e-06,
+      "loss": 0.1048,
+      "step": 427
+    },
+    {
+      "epoch": 3.8963414634146343,
+      "grad_norm": 0.8066243529319763,
+      "learning_rate": 1.7870170129584191e-06,
+      "loss": 0.1017,
+      "step": 428
+    },
+    {
+      "epoch": 3.9054878048780486,
+      "grad_norm": 0.6243605613708496,
+      "learning_rate": 1.7734404517597004e-06,
+      "loss": 0.1048,
+      "step": 429
+    },
+    {
+      "epoch": 3.9146341463414633,
+      "grad_norm": 0.7325178384780884,
+      "learning_rate": 1.7598872547260904e-06,
+      "loss": 0.1018,
+      "step": 430
+    },
+    {
+      "epoch": 3.923780487804878,
+      "grad_norm": 0.8298372030258179,
+      "learning_rate": 1.7463578576912859e-06,
+      "loss": 0.1061,
+      "step": 431
+    },
+    {
+      "epoch": 3.932926829268293,
+      "grad_norm": 0.3550826609134674,
+      "learning_rate": 1.7328526957236408e-06,
+      "loss": 0.105,
+      "step": 432
+    },
+    {
+      "epoch": 3.942073170731707,
+      "grad_norm": 0.6778271794319153,
+      "learning_rate": 1.7193722031121726e-06,
+      "loss": 0.1007,
+      "step": 433
+    },
+    {
+      "epoch": 3.951219512195122,
+      "grad_norm": 0.8127948641777039,
+      "learning_rate": 1.7059168133526043e-06,
+      "loss": 0.1009,
+      "step": 434
+    },
+    {
+      "epoch": 3.9603658536585367,
+      "grad_norm": 0.3050505518913269,
+      "learning_rate": 1.692486959133417e-06,
+      "loss": 0.1035,
+      "step": 435
+    },
+    {
+      "epoch": 3.9695121951219514,
+      "grad_norm": 0.6452750563621521,
+      "learning_rate": 1.6790830723219398e-06,
+      "loss": 0.1061,
+      "step": 436
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 654,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 6,
+  "save_steps": 109,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.3172242912789922e+19,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}
diff --git a/checkpoint-436/training_args.bin b/checkpoint-436/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..24b9be216f1966bb6d081a79c30a21dced39cf99
--- /dev/null
+++ b/checkpoint-436/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:14990b8399bb8d39b746405b094f7eea1041d93488de8433b2d415de16ff957a
+size 7928
diff --git a/checkpoint-436/zero_to_fp32.py b/checkpoint-436/zero_to_fp32.py
new file mode 100644
index 0000000000000000000000000000000000000000..24cc342e78d1a006c782b3a4cd68d9ce786d8fd8
--- /dev/null
+++ b/checkpoint-436/zero_to_fp32.py
@@ -0,0 +1,604 @@
+#!/usr/bin/env python
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
+# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
+# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
+# application.
+#
+# example: python zero_to_fp32.py . pytorch_model.bin
+
+import argparse
+import torch
+import glob
+import math
+import os
+import re
+from collections import OrderedDict
+from dataclasses import dataclass
+
+# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
+# DeepSpeed data structures it has to be available in the current python environment.
+from deepspeed.utils import logger
+from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
+                                            FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
+                                            FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS)
+
+
+@dataclass
+class zero_model_state:
+    buffers: dict()
+    param_shapes: dict()
+    shared_params: list
+    ds_version: int
+    frozen_param_shapes: dict()
+    frozen_param_fragments: dict()
+
+
+debug = 0
+
+# load to cpu
+device = torch.device('cpu')
+
+
+def atoi(text):
+    return int(text) if text.isdigit() else text
+
+
+def natural_keys(text):
+    '''
+    alist.sort(key=natural_keys) sorts in human order
+    http://nedbatchelder.com/blog/200712/human_sorting.html
+    (See Toothy's implementation in the comments)
+    '''
+    return [atoi(c) for c in re.split(r'(\d+)', text)]
+
+
+def get_model_state_file(checkpoint_dir, zero_stage):
+    if not os.path.isdir(checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
+
+    # there should be only one file
+    if zero_stage <= 2:
+        file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
+    elif zero_stage == 3:
+        file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
+
+    if not os.path.exists(file):
+        raise FileNotFoundError(f"can't find model states file at '{file}'")
+
+    return file
+
+
+def get_checkpoint_files(checkpoint_dir, glob_pattern):
+    # XXX: need to test that this simple glob rule works for multi-node setup too
+    ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
+
+    if len(ckpt_files) == 0:
+        raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
+
+    return ckpt_files
+
+
+def get_optim_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
+
+
+def get_model_state_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
+
+
+def parse_model_states(files):
+    zero_model_states = []
+    for file in files:
+        state_dict = torch.load(file, map_location=device)
+
+        if BUFFER_NAMES not in state_dict:
+            raise ValueError(f"{file} is not a model state checkpoint")
+        buffer_names = state_dict[BUFFER_NAMES]
+        if debug:
+            print("Found buffers:", buffer_names)
+
+        # recover just the buffers while restoring them to fp32 if they were saved in fp16
+        buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
+        param_shapes = state_dict[PARAM_SHAPES]
+
+        # collect parameters that are included in param_shapes
+        param_names = []
+        for s in param_shapes:
+            for name in s.keys():
+                param_names.append(name)
+
+        # update with frozen parameters
+        frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
+        if frozen_param_shapes is not None:
+            if debug:
+                print(f"Found frozen_param_shapes: {frozen_param_shapes}")
+            param_names += list(frozen_param_shapes.keys())
+
+        # handle shared params
+        shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
+
+        ds_version = state_dict.get(DS_VERSION, None)
+
+        frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
+
+        z_model_state = zero_model_state(buffers=buffers,
+                                         param_shapes=param_shapes,
+                                         shared_params=shared_params,
+                                         ds_version=ds_version,
+                                         frozen_param_shapes=frozen_param_shapes,
+                                         frozen_param_fragments=frozen_param_fragments)
+        zero_model_states.append(z_model_state)
+
+    return zero_model_states
+
+
+def parse_optim_states(files, ds_checkpoint_dir):
+
+    total_files = len(files)
+    state_dicts = []
+    for f in files:
+        state_dict = torch.load(f, map_location=device)
+        # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+        # and also handle the case where it was already removed by another helper script
+        state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+        state_dicts.append(state_dict)
+
+    if not ZERO_STAGE in state_dicts[0][OPTIMIZER_STATE_DICT]:
+        raise ValueError(f"{files[0]} is not a zero checkpoint")
+    zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
+    world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
+
+    # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
+    # parameters can be different from data parallelism for non-expert parameters. So we can just
+    # use the max of the partition_count to get the dp world_size.
+
+    if type(world_size) is list:
+        world_size = max(world_size)
+
+    if world_size != total_files:
+        raise ValueError(
+            f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
+            "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
+        )
+
+    # the groups are named differently in each stage
+    if zero_stage <= 2:
+        fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
+    elif zero_stage == 3:
+        fp32_groups_key = FP32_FLAT_GROUPS
+    else:
+        raise ValueError(f"unknown zero stage {zero_stage}")
+
+    if zero_stage <= 2:
+        fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
+    elif zero_stage == 3:
+        # if there is more than one param group, there will be multiple flattened tensors - one
+        # flattened tensor per group - for simplicity merge them into a single tensor
+        #
+        # XXX: could make the script more memory efficient for when there are multiple groups - it
+        # will require matching the sub-lists of param_shapes for each param group flattened tensor
+
+        fp32_flat_groups = [
+            torch.cat(state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key], 0) for i in range(len(state_dicts))
+        ]
+
+    return zero_stage, world_size, fp32_flat_groups
+
+
+def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
+    """
+    Returns fp32 state_dict reconstructed from ds checkpoint
+
+    Args:
+        - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
+
+    """
+    print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
+
+    optim_files = get_optim_files(ds_checkpoint_dir)
+    zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
+    print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
+
+    model_files = get_model_state_files(ds_checkpoint_dir)
+
+    zero_model_states = parse_model_states(model_files)
+    print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
+
+    if zero_stage <= 2:
+        return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+    elif zero_stage == 3:
+        return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+
+
+def _zero2_merge_frozen_params(state_dict, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+    frozen_param_fragments = zero_model_states[0].frozen_param_fragments
+
+    if debug:
+        num_elem = sum(s.numel() for s in frozen_param_shapes.values())
+        print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        state_dict[name] = frozen_param_fragments[name]
+
+        if debug:
+            print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _has_callable(obj, fn):
+    attr = getattr(obj, fn, None)
+    return callable(attr)
+
+
+def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+
+    # Reconstruction protocol:
+    #
+    # XXX: document this
+
+    if debug:
+        for i in range(world_size):
+            for j in range(len(fp32_flat_groups[0])):
+                print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
+
+    # XXX: memory usage doubles here (zero2)
+    num_param_groups = len(fp32_flat_groups[0])
+    merged_single_partition_of_fp32_groups = []
+    for i in range(num_param_groups):
+        merged_partitions = [sd[i] for sd in fp32_flat_groups]
+        full_single_fp32_vector = torch.cat(merged_partitions, 0)
+        merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
+    avail_numel = sum(
+        [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
+
+    if debug:
+        wanted_params = sum([len(shapes) for shapes in param_shapes])
+        wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
+        # not asserting if there is a mismatch due to possible padding
+        print(f"Have {avail_numel} numels to process.")
+        print(f"Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    total_numel = 0
+    total_params = 0
+    for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
+        offset = 0
+        avail_numel = full_single_fp32_vector.numel()
+        for name, shape in shapes.items():
+
+            unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
+            total_numel += unpartitioned_numel
+            total_params += 1
+
+            if debug:
+                print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+            state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
+            offset += unpartitioned_numel
+
+        # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
+        # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
+        # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
+        # live optimizer object, so we are checking that the numbers are within the right range
+        align_to = 2 * world_size
+
+        def zero2_align(x):
+            return align_to * math.ceil(x / align_to)
+
+        if debug:
+            print(f"original offset={offset}, avail_numel={avail_numel}")
+
+        offset = zero2_align(offset)
+        avail_numel = zero2_align(avail_numel)
+
+        if debug:
+            print(f"aligned  offset={offset}, avail_numel={avail_numel}")
+
+        # Sanity check
+        if offset != avail_numel:
+            raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero2_merge_frozen_params(state_dict, zero_model_states)
+
+    _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def zero3_partitioned_param_info(unpartitioned_numel, world_size):
+    remainder = unpartitioned_numel % world_size
+    padding_numel = (world_size - remainder) if remainder else 0
+    partitioned_numel = math.ceil(unpartitioned_numel / world_size)
+    return partitioned_numel, padding_numel
+
+
+def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    if debug:
+        for i in range(world_size):
+            num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
+            print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in zero_model_states[0].frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
+        state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
+
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+    avail_numel = fp32_flat_groups[0].numel() * world_size
+    # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
+    # param, re-consolidating each param, while dealing with padding if any
+
+    # merge list of dicts, preserving order
+    param_shapes = {k: v for d in param_shapes for k, v in d.items()}
+
+    if debug:
+        for i in range(world_size):
+            print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
+
+        wanted_params = len(param_shapes)
+        wanted_numel = sum(shape.numel() for shape in param_shapes.values())
+        # not asserting if there is a mismatch due to possible padding
+        avail_numel = fp32_flat_groups[0].numel() * world_size
+        print(f"Trainable params: Have {avail_numel} numels to process.")
+        print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    offset = 0
+    total_numel = 0
+    total_params = 0
+    for name, shape in param_shapes.items():
+
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+        total_params += 1
+
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+        # XXX: memory usage doubles here
+        state_dict[name] = torch.cat(
+            tuple(fp32_flat_groups[i].narrow(0, offset, partitioned_numel) for i in range(world_size)),
+            0).narrow(0, 0, unpartitioned_numel).view(shape)
+        offset += partitioned_numel
+
+    offset *= world_size
+
+    # Sanity check
+    if offset != avail_numel:
+        raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
+
+    _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag=None, exclude_frozen_parameters=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
+    ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
+    via a model hub.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+
+    Returns:
+        - pytorch ``state_dict``
+
+    Note: this approach may not work if your application doesn't have sufficient free CPU memory and
+    you may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
+    the checkpoint.
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+        # do the training and checkpoint saving
+        state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
+        model = model.cpu() # move to cpu
+        model.load_state_dict(state_dict)
+        # submit to model hub or save the model to share with others
+
+    In this example the ``model`` will no longer be usable in the deepspeed context of the same
+    application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
+
+    """
+    if tag is None:
+        latest_path = os.path.join(checkpoint_dir, 'latest')
+        if os.path.isfile(latest_path):
+            with open(latest_path, 'r') as fd:
+                tag = fd.read().strip()
+        else:
+            raise ValueError(f"Unable to find 'latest' file at {latest_path}")
+
+    ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
+
+    if not os.path.isdir(ds_checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
+
+    return _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
+
+
+def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, output_file, tag=None, exclude_frozen_parameters=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
+    loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``output_file``: path to the pytorch fp32 state_dict output file (e.g. path/pytorch_model.bin)
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+    """
+
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag, exclude_frozen_parameters)
+    print(f"Saving fp32 state dict to {output_file}")
+    torch.save(state_dict, output_file)
+
+
+def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
+    """
+    1. Put the provided model to cpu
+    2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
+    3. Load it into the provided model
+
+    Args:
+        - ``model``: the model object to update
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+
+    Returns:
+        - ``model`: modified model
+
+    Make sure you have plenty of CPU memory available before you call this function. If you don't
+    have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
+    conveniently placed for you in the checkpoint folder.
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
+        model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
+        # submit to model hub or save the model to share with others
+
+    Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
+    of the same application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    """
+    logger.info(f"Extracting fp32 weights")
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+
+    logger.info(f"Overwriting model with fp32 weights")
+    model = model.cpu()
+    model.load_state_dict(state_dict, strict=False)
+
+    return model
+
+
+if __name__ == "__main__":
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("checkpoint_dir",
+                        type=str,
+                        help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+    parser.add_argument(
+        "output_file",
+        type=str,
+        help="path to the pytorch fp32 state_dict output file (e.g. path/checkpoint-12/pytorch_model.bin)")
+    parser.add_argument("-t",
+                        "--tag",
+                        type=str,
+                        default=None,
+                        help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
+    parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+    parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
+    args = parser.parse_args()
+
+    debug = args.debug
+
+    convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
+                                               args.output_file,
+                                               tag=args.tag,
+                                               exclude_frozen_parameters=args.exclude_frozen_parameters)
diff --git a/checkpoint-545/README.md b/checkpoint-545/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..f4a3934800eeb082a0cb833d7b6af4f68eed3615
--- /dev/null
+++ b/checkpoint-545/README.md
@@ -0,0 +1,202 @@
+---
+base_model: nvidia/Llama-3_3-Nemotron-Super-49B-v1
+library_name: peft
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.15.0
\ No newline at end of file
diff --git a/checkpoint-545/adapter_config.json b/checkpoint-545/adapter_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..d7d7c196589a89c5b284ba682f27212262154f44
--- /dev/null
+++ b/checkpoint-545/adapter_config.json
@@ -0,0 +1,42 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "nvidia/Llama-3_3-Nemotron-Super-49B-v1",
+  "bias": "none",
+  "corda_config": null,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": null,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 512,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": [
+    "embed_tokens",
+    "lm_head"
+  ],
+  "peft_type": "LORA",
+  "r": 256,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "o_proj",
+    "gate_proj",
+    "q_proj",
+    "k_proj",
+    "v_proj",
+    "up_proj",
+    "down_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_rslora": false
+}
\ No newline at end of file
diff --git a/checkpoint-545/adapter_model.safetensors b/checkpoint-545/adapter_model.safetensors
new file mode 100644
index 0000000000000000000000000000000000000000..08a98b97d5f65cd2d9850930fc4f3208477a4bbb
--- /dev/null
+++ b/checkpoint-545/adapter_model.safetensors
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d0df1ebbc3fec39d590ce612b0a58365d356d4e289b303fdf9555ff878279a96
+size 9016826528
diff --git a/checkpoint-545/global_step542/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt b/checkpoint-545/global_step542/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..757a10b6853b915b5020769b1be63a65b522875e
--- /dev/null
+++ b/checkpoint-545/global_step542/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c37a7af31519b6433daf918f3dbad7de27a06c7615ff0ccbaed04144fd7bffce
+size 27050164444
diff --git a/checkpoint-545/global_step542/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt b/checkpoint-545/global_step542/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..57fb970a6f96589a1a1796a59cebff33161d1be9
--- /dev/null
+++ b/checkpoint-545/global_step542/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:18429fb1e1c4b721916c4180636129f33991f310d5f79ff9c2603c5094143279
+size 27050169884
diff --git a/checkpoint-545/global_step542/mp_rank_00_model_states.pt b/checkpoint-545/global_step542/mp_rank_00_model_states.pt
new file mode 100644
index 0000000000000000000000000000000000000000..01e4da8d64403fd6a36bf2e6ebbfc42059cbf33f
--- /dev/null
+++ b/checkpoint-545/global_step542/mp_rank_00_model_states.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a163ce838e765d5a6059f3157857e8fb2a09ab506ea2932e8f4715a5f249f7df
+size 9776788601
diff --git a/checkpoint-545/latest b/checkpoint-545/latest
new file mode 100644
index 0000000000000000000000000000000000000000..a05308d379be76c1fc44543d3567916349dc9880
--- /dev/null
+++ b/checkpoint-545/latest
@@ -0,0 +1 @@
+global_step542
\ No newline at end of file
diff --git a/checkpoint-545/rng_state_0.pth b/checkpoint-545/rng_state_0.pth
new file mode 100644
index 0000000000000000000000000000000000000000..b109f54663c252501dbe9a18a3fb70a115540e7f
--- /dev/null
+++ b/checkpoint-545/rng_state_0.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3f710ae0c78097f0623dea0915c2272859eaf5aa9521583f64ebf5ee4aae67cf
+size 14512
diff --git a/checkpoint-545/rng_state_1.pth b/checkpoint-545/rng_state_1.pth
new file mode 100644
index 0000000000000000000000000000000000000000..5fc30c40f4e4c35f4f80a009a2f3c4eec6ffa8f9
--- /dev/null
+++ b/checkpoint-545/rng_state_1.pth
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a06e175f50a874c36eb85bf795ed23a055c0f59cf2995ab8043aad4489ba348
+size 14512
diff --git a/checkpoint-545/scheduler.pt b/checkpoint-545/scheduler.pt
new file mode 100644
index 0000000000000000000000000000000000000000..09ab4bbbcd4101821c9b86534a4a9a1e5a33c3b6
--- /dev/null
+++ b/checkpoint-545/scheduler.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2a35b1b850ca4cf72aba300ca7bb58349b271c9cca421450336560876d667241
+size 1064
diff --git a/checkpoint-545/special_tokens_map.json b/checkpoint-545/special_tokens_map.json
new file mode 100644
index 0000000000000000000000000000000000000000..278b7f0f84be865c4687700ee7b3c63d89a51e18
--- /dev/null
+++ b/checkpoint-545/special_tokens_map.json
@@ -0,0 +1,23 @@
+{
+  "bos_token": {
+    "content": "<|begin_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|eot_id|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|end_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}
diff --git a/checkpoint-545/tokenizer.json b/checkpoint-545/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..1c1d8d5c9024994f1d3b00f9662b8dd89ca13cf2
--- /dev/null
+++ b/checkpoint-545/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920
diff --git a/checkpoint-545/tokenizer_config.json b/checkpoint-545/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..edd01b980c1db496ea102a51c972ee8f5d1a2c74
--- /dev/null
+++ b/checkpoint-545/tokenizer_config.json
@@ -0,0 +1,2064 @@
+{
+  "added_tokens_decoder": {
+    "128000": {
+      "content": "<|begin_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128001": {
+      "content": "<|end_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128002": {
+      "content": "<|reserved_special_token_0|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128003": {
+      "content": "<|reserved_special_token_1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128004": {
+      "content": "<|finetune_right_pad_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128005": {
+      "content": "<|reserved_special_token_2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128006": {
+      "content": "<|start_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128007": {
+      "content": "<|end_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128008": {
+      "content": "<|eom_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128009": {
+      "content": "<|eot_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128010": {
+      "content": "<|python_tag|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128011": {
+      "content": "<|reserved_special_token_3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128012": {
+      "content": "<|reserved_special_token_4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128013": {
+      "content": "<|reserved_special_token_5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128014": {
+      "content": "<|reserved_special_token_6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128015": {
+      "content": "<|reserved_special_token_7|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128016": {
+      "content": "<|reserved_special_token_8|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128017": {
+      "content": "<|reserved_special_token_9|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128018": {
+      "content": "<|reserved_special_token_10|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128019": {
+      "content": "<|reserved_special_token_11|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128020": {
+      "content": "<|reserved_special_token_12|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128021": {
+      "content": "<|reserved_special_token_13|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128022": {
+      "content": "<|reserved_special_token_14|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128023": {
+      "content": "<|reserved_special_token_15|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128024": {
+      "content": "<|reserved_special_token_16|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128025": {
+      "content": "<|reserved_special_token_17|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128026": {
+      "content": "<|reserved_special_token_18|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128027": {
+      "content": "<|reserved_special_token_19|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128028": {
+      "content": "<|reserved_special_token_20|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128029": {
+      "content": "<|reserved_special_token_21|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128030": {
+      "content": "<|reserved_special_token_22|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128031": {
+      "content": "<|reserved_special_token_23|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128032": {
+      "content": "<|reserved_special_token_24|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128033": {
+      "content": "<|reserved_special_token_25|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128034": {
+      "content": "<|reserved_special_token_26|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128035": {
+      "content": "<|reserved_special_token_27|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128036": {
+      "content": "<|reserved_special_token_28|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128037": {
+      "content": "<|reserved_special_token_29|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128038": {
+      "content": "<|reserved_special_token_30|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128039": {
+      "content": "<|reserved_special_token_31|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128040": {
+      "content": "<|reserved_special_token_32|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128041": {
+      "content": "<|reserved_special_token_33|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128042": {
+      "content": "<|reserved_special_token_34|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128043": {
+      "content": "<|reserved_special_token_35|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128044": {
+      "content": "<|reserved_special_token_36|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128045": {
+      "content": "<|reserved_special_token_37|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128046": {
+      "content": "<|reserved_special_token_38|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128047": {
+      "content": "<|reserved_special_token_39|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128048": {
+      "content": "<|reserved_special_token_40|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128049": {
+      "content": "<|reserved_special_token_41|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128050": {
+      "content": "<|reserved_special_token_42|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128051": {
+      "content": "<|reserved_special_token_43|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128052": {
+      "content": "<|reserved_special_token_44|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128053": {
+      "content": "<|reserved_special_token_45|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128054": {
+      "content": "<|reserved_special_token_46|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128055": {
+      "content": "<|reserved_special_token_47|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128056": {
+      "content": "<|reserved_special_token_48|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128057": {
+      "content": "<|reserved_special_token_49|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128058": {
+      "content": "<|reserved_special_token_50|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128059": {
+      "content": "<|reserved_special_token_51|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128060": {
+      "content": "<|reserved_special_token_52|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128061": {
+      "content": "<|reserved_special_token_53|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128062": {
+      "content": "<|reserved_special_token_54|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128063": {
+      "content": "<|reserved_special_token_55|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128064": {
+      "content": "<|reserved_special_token_56|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128065": {
+      "content": "<|reserved_special_token_57|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128066": {
+      "content": "<|reserved_special_token_58|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128067": {
+      "content": "<|reserved_special_token_59|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128068": {
+      "content": "<|reserved_special_token_60|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128069": {
+      "content": "<|reserved_special_token_61|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128070": {
+      "content": "<|reserved_special_token_62|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128071": {
+      "content": "<|reserved_special_token_63|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128072": {
+      "content": "<|reserved_special_token_64|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128073": {
+      "content": "<|reserved_special_token_65|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128074": {
+      "content": "<|reserved_special_token_66|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128075": {
+      "content": "<|reserved_special_token_67|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128076": {
+      "content": "<|reserved_special_token_68|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128077": {
+      "content": "<|reserved_special_token_69|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128078": {
+      "content": "<|reserved_special_token_70|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128079": {
+      "content": "<|reserved_special_token_71|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128080": {
+      "content": "<|reserved_special_token_72|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128081": {
+      "content": "<|reserved_special_token_73|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128082": {
+      "content": "<|reserved_special_token_74|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128083": {
+      "content": "<|reserved_special_token_75|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128084": {
+      "content": "<|reserved_special_token_76|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128085": {
+      "content": "<|reserved_special_token_77|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128086": {
+      "content": "<|reserved_special_token_78|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128087": {
+      "content": "<|reserved_special_token_79|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128088": {
+      "content": "<|reserved_special_token_80|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128089": {
+      "content": "<|reserved_special_token_81|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128090": {
+      "content": "<|reserved_special_token_82|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128091": {
+      "content": "<|reserved_special_token_83|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128092": {
+      "content": "<|reserved_special_token_84|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128093": {
+      "content": "<|reserved_special_token_85|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128094": {
+      "content": "<|reserved_special_token_86|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128095": {
+      "content": "<|reserved_special_token_87|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128096": {
+      "content": "<|reserved_special_token_88|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128097": {
+      "content": "<|reserved_special_token_89|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128098": {
+      "content": "<|reserved_special_token_90|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128099": {
+      "content": "<|reserved_special_token_91|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128100": {
+      "content": "<|reserved_special_token_92|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128101": {
+      "content": "<|reserved_special_token_93|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128102": {
+      "content": "<|reserved_special_token_94|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128103": {
+      "content": "<|reserved_special_token_95|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128104": {
+      "content": "<|reserved_special_token_96|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128105": {
+      "content": "<|reserved_special_token_97|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128106": {
+      "content": "<|reserved_special_token_98|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128107": {
+      "content": "<|reserved_special_token_99|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128108": {
+      "content": "<|reserved_special_token_100|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128109": {
+      "content": "<|reserved_special_token_101|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128110": {
+      "content": "<|reserved_special_token_102|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128111": {
+      "content": "<|reserved_special_token_103|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128112": {
+      "content": "<|reserved_special_token_104|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128113": {
+      "content": "<|reserved_special_token_105|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128114": {
+      "content": "<|reserved_special_token_106|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128115": {
+      "content": "<|reserved_special_token_107|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128116": {
+      "content": "<|reserved_special_token_108|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128117": {
+      "content": "<|reserved_special_token_109|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128118": {
+      "content": "<|reserved_special_token_110|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128119": {
+      "content": "<|reserved_special_token_111|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128120": {
+      "content": "<|reserved_special_token_112|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128121": {
+      "content": "<|reserved_special_token_113|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128122": {
+      "content": "<|reserved_special_token_114|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128123": {
+      "content": "<|reserved_special_token_115|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128124": {
+      "content": "<|reserved_special_token_116|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128125": {
+      "content": "<|reserved_special_token_117|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128126": {
+      "content": "<|reserved_special_token_118|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128127": {
+      "content": "<|reserved_special_token_119|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128128": {
+      "content": "<|reserved_special_token_120|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128129": {
+      "content": "<|reserved_special_token_121|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128130": {
+      "content": "<|reserved_special_token_122|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128131": {
+      "content": "<|reserved_special_token_123|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128132": {
+      "content": "<|reserved_special_token_124|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128133": {
+      "content": "<|reserved_special_token_125|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128134": {
+      "content": "<|reserved_special_token_126|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128135": {
+      "content": "<|reserved_special_token_127|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128136": {
+      "content": "<|reserved_special_token_128|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128137": {
+      "content": "<|reserved_special_token_129|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128138": {
+      "content": "<|reserved_special_token_130|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128139": {
+      "content": "<|reserved_special_token_131|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128140": {
+      "content": "<|reserved_special_token_132|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128141": {
+      "content": "<|reserved_special_token_133|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128142": {
+      "content": "<|reserved_special_token_134|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128143": {
+      "content": "<|reserved_special_token_135|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128144": {
+      "content": "<|reserved_special_token_136|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128145": {
+      "content": "<|reserved_special_token_137|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128146": {
+      "content": "<|reserved_special_token_138|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128147": {
+      "content": "<|reserved_special_token_139|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128148": {
+      "content": "<|reserved_special_token_140|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128149": {
+      "content": "<|reserved_special_token_141|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128150": {
+      "content": "<|reserved_special_token_142|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128151": {
+      "content": "<|reserved_special_token_143|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128152": {
+      "content": "<|reserved_special_token_144|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128153": {
+      "content": "<|reserved_special_token_145|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128154": {
+      "content": "<|reserved_special_token_146|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128155": {
+      "content": "<|reserved_special_token_147|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128156": {
+      "content": "<|reserved_special_token_148|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128157": {
+      "content": "<|reserved_special_token_149|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128158": {
+      "content": "<|reserved_special_token_150|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128159": {
+      "content": "<|reserved_special_token_151|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128160": {
+      "content": "<|reserved_special_token_152|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128161": {
+      "content": "<|reserved_special_token_153|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128162": {
+      "content": "<|reserved_special_token_154|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128163": {
+      "content": "<|reserved_special_token_155|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128164": {
+      "content": "<|reserved_special_token_156|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128165": {
+      "content": "<|reserved_special_token_157|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128166": {
+      "content": "<|reserved_special_token_158|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128167": {
+      "content": "<|reserved_special_token_159|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128168": {
+      "content": "<|reserved_special_token_160|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128169": {
+      "content": "<|reserved_special_token_161|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128170": {
+      "content": "<|reserved_special_token_162|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128171": {
+      "content": "<|reserved_special_token_163|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128172": {
+      "content": "<|reserved_special_token_164|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128173": {
+      "content": "<|reserved_special_token_165|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128174": {
+      "content": "<|reserved_special_token_166|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128175": {
+      "content": "<|reserved_special_token_167|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128176": {
+      "content": "<|reserved_special_token_168|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128177": {
+      "content": "<|reserved_special_token_169|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128178": {
+      "content": "<|reserved_special_token_170|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128179": {
+      "content": "<|reserved_special_token_171|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128180": {
+      "content": "<|reserved_special_token_172|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128181": {
+      "content": "<|reserved_special_token_173|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128182": {
+      "content": "<|reserved_special_token_174|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128183": {
+      "content": "<|reserved_special_token_175|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128184": {
+      "content": "<|reserved_special_token_176|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128185": {
+      "content": "<|reserved_special_token_177|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128186": {
+      "content": "<|reserved_special_token_178|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128187": {
+      "content": "<|reserved_special_token_179|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128188": {
+      "content": "<|reserved_special_token_180|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128189": {
+      "content": "<|reserved_special_token_181|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128190": {
+      "content": "<|reserved_special_token_182|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128191": {
+      "content": "<|reserved_special_token_183|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128192": {
+      "content": "<|reserved_special_token_184|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128193": {
+      "content": "<|reserved_special_token_185|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128194": {
+      "content": "<|reserved_special_token_186|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128195": {
+      "content": "<|reserved_special_token_187|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128196": {
+      "content": "<|reserved_special_token_188|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128197": {
+      "content": "<|reserved_special_token_189|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128198": {
+      "content": "<|reserved_special_token_190|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128199": {
+      "content": "<|reserved_special_token_191|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128200": {
+      "content": "<|reserved_special_token_192|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128201": {
+      "content": "<|reserved_special_token_193|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128202": {
+      "content": "<|reserved_special_token_194|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128203": {
+      "content": "<|reserved_special_token_195|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128204": {
+      "content": "<|reserved_special_token_196|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128205": {
+      "content": "<|reserved_special_token_197|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128206": {
+      "content": "<|reserved_special_token_198|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128207": {
+      "content": "<|reserved_special_token_199|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128208": {
+      "content": "<|reserved_special_token_200|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128209": {
+      "content": "<|reserved_special_token_201|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128210": {
+      "content": "<|reserved_special_token_202|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128211": {
+      "content": "<|reserved_special_token_203|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128212": {
+      "content": "<|reserved_special_token_204|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128213": {
+      "content": "<|reserved_special_token_205|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128214": {
+      "content": "<|reserved_special_token_206|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128215": {
+      "content": "<|reserved_special_token_207|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128216": {
+      "content": "<|reserved_special_token_208|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128217": {
+      "content": "<|reserved_special_token_209|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128218": {
+      "content": "<|reserved_special_token_210|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128219": {
+      "content": "<|reserved_special_token_211|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128220": {
+      "content": "<|reserved_special_token_212|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128221": {
+      "content": "<|reserved_special_token_213|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128222": {
+      "content": "<|reserved_special_token_214|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128223": {
+      "content": "<|reserved_special_token_215|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128224": {
+      "content": "<|reserved_special_token_216|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128225": {
+      "content": "<|reserved_special_token_217|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128226": {
+      "content": "<|reserved_special_token_218|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128227": {
+      "content": "<|reserved_special_token_219|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128228": {
+      "content": "<|reserved_special_token_220|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128229": {
+      "content": "<|reserved_special_token_221|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128230": {
+      "content": "<|reserved_special_token_222|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128231": {
+      "content": "<|reserved_special_token_223|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128232": {
+      "content": "<|reserved_special_token_224|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128233": {
+      "content": "<|reserved_special_token_225|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128234": {
+      "content": "<|reserved_special_token_226|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128235": {
+      "content": "<|reserved_special_token_227|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128236": {
+      "content": "<|reserved_special_token_228|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128237": {
+      "content": "<|reserved_special_token_229|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128238": {
+      "content": "<|reserved_special_token_230|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128239": {
+      "content": "<|reserved_special_token_231|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128240": {
+      "content": "<|reserved_special_token_232|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128241": {
+      "content": "<|reserved_special_token_233|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128242": {
+      "content": "<|reserved_special_token_234|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128243": {
+      "content": "<|reserved_special_token_235|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128244": {
+      "content": "<|reserved_special_token_236|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128245": {
+      "content": "<|reserved_special_token_237|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128246": {
+      "content": "<|reserved_special_token_238|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128247": {
+      "content": "<|reserved_special_token_239|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128248": {
+      "content": "<|reserved_special_token_240|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128249": {
+      "content": "<|reserved_special_token_241|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128250": {
+      "content": "<|reserved_special_token_242|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128251": {
+      "content": "<|reserved_special_token_243|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128252": {
+      "content": "<|reserved_special_token_244|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128253": {
+      "content": "<|reserved_special_token_245|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128254": {
+      "content": "<|reserved_special_token_246|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128255": {
+      "content": "<|reserved_special_token_247|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|begin_of_text|>",
+  "chat_template": "{{- bos_token }}{%- if messages[0]['role'] == 'system' %}{%- set system_message = messages[0]['content']|trim %}{%- set messages = messages[1:] %}{%- else %}{%- set system_message = \"\" %}{%- endif %}{{- \"<|start_header_id|>system<|end_header_id|>\\n\\n\" }}{{- system_message }}{{- \"<|eot_id|>\" }}{%- for message in messages %}{%- if message['role'] == 'assistant' and '</think>' in message['content'] %}{%- set content = message['content'].split('</think>')[-1].lstrip() %}{%- else %}{%- set content = message['content'] %}{%- endif %}{{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\\n\\n' + content | trim + '<|eot_id|>' }}{%- endfor %}{%- if add_generation_prompt %}{{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' }}{%- endif %}",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|eot_id|>",
+  "extra_special_tokens": {},
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "pad_token": "<|end_of_text|>",
+  "tokenizer_class": "PreTrainedTokenizer"
+}
diff --git a/checkpoint-545/trainer_state.json b/checkpoint-545/trainer_state.json
new file mode 100644
index 0000000000000000000000000000000000000000..dfa225c6dc04b8eeb583f716d0b5d90ae095148d
--- /dev/null
+++ b/checkpoint-545/trainer_state.json
@@ -0,0 +1,3848 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 4.960365853658536,
+  "eval_steps": 500,
+  "global_step": 545,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.009146341463414634,
+      "grad_norm": 40.05127716064453,
+      "learning_rate": 5.0000000000000004e-08,
+      "loss": 5.1282,
+      "step": 1
+    },
+    {
+      "epoch": 0.018292682926829267,
+      "grad_norm": 39.01039505004883,
+      "learning_rate": 1.0000000000000001e-07,
+      "loss": 5.0586,
+      "step": 2
+    },
+    {
+      "epoch": 0.027439024390243903,
+      "grad_norm": 40.20766067504883,
+      "learning_rate": 1.5000000000000002e-07,
+      "loss": 5.0311,
+      "step": 3
+    },
+    {
+      "epoch": 0.036585365853658534,
+      "grad_norm": 38.70350646972656,
+      "learning_rate": 2.0000000000000002e-07,
+      "loss": 5.1598,
+      "step": 4
+    },
+    {
+      "epoch": 0.04573170731707317,
+      "grad_norm": 39.230979919433594,
+      "learning_rate": 2.5000000000000004e-07,
+      "loss": 5.0812,
+      "step": 5
+    },
+    {
+      "epoch": 0.054878048780487805,
+      "grad_norm": 39.45293045043945,
+      "learning_rate": 3.0000000000000004e-07,
+      "loss": 5.0841,
+      "step": 6
+    },
+    {
+      "epoch": 0.06402439024390244,
+      "grad_norm": 38.92170333862305,
+      "learning_rate": 3.5000000000000004e-07,
+      "loss": 5.022,
+      "step": 7
+    },
+    {
+      "epoch": 0.07317073170731707,
+      "grad_norm": 38.932369232177734,
+      "learning_rate": 4.0000000000000003e-07,
+      "loss": 5.0876,
+      "step": 8
+    },
+    {
+      "epoch": 0.08231707317073171,
+      "grad_norm": 39.60245132446289,
+      "learning_rate": 4.5000000000000003e-07,
+      "loss": 5.0798,
+      "step": 9
+    },
+    {
+      "epoch": 0.09146341463414634,
+      "grad_norm": 39.53607177734375,
+      "learning_rate": 5.000000000000001e-07,
+      "loss": 5.0105,
+      "step": 10
+    },
+    {
+      "epoch": 0.10060975609756098,
+      "grad_norm": 39.50700378417969,
+      "learning_rate": 5.5e-07,
+      "loss": 4.9951,
+      "step": 11
+    },
+    {
+      "epoch": 0.10975609756097561,
+      "grad_norm": 39.52376174926758,
+      "learning_rate": 6.000000000000001e-07,
+      "loss": 4.9163,
+      "step": 12
+    },
+    {
+      "epoch": 0.11890243902439024,
+      "grad_norm": 39.15214157104492,
+      "learning_rate": 6.5e-07,
+      "loss": 4.9634,
+      "step": 13
+    },
+    {
+      "epoch": 0.12804878048780488,
+      "grad_norm": 37.795066833496094,
+      "learning_rate": 7.000000000000001e-07,
+      "loss": 4.819,
+      "step": 14
+    },
+    {
+      "epoch": 0.13719512195121952,
+      "grad_norm": 38.23014450073242,
+      "learning_rate": 7.5e-07,
+      "loss": 4.8756,
+      "step": 15
+    },
+    {
+      "epoch": 0.14634146341463414,
+      "grad_norm": 38.57551574707031,
+      "learning_rate": 8.000000000000001e-07,
+      "loss": 4.81,
+      "step": 16
+    },
+    {
+      "epoch": 0.15548780487804878,
+      "grad_norm": 36.06121063232422,
+      "learning_rate": 8.500000000000001e-07,
+      "loss": 4.7901,
+      "step": 17
+    },
+    {
+      "epoch": 0.16463414634146342,
+      "grad_norm": 34.38008117675781,
+      "learning_rate": 9.000000000000001e-07,
+      "loss": 4.5775,
+      "step": 18
+    },
+    {
+      "epoch": 0.17378048780487804,
+      "grad_norm": 32.60890579223633,
+      "learning_rate": 9.500000000000001e-07,
+      "loss": 4.3737,
+      "step": 19
+    },
+    {
+      "epoch": 0.18292682926829268,
+      "grad_norm": 31.983110427856445,
+      "learning_rate": 1.0000000000000002e-06,
+      "loss": 4.3773,
+      "step": 20
+    },
+    {
+      "epoch": 0.19207317073170732,
+      "grad_norm": 27.69571876525879,
+      "learning_rate": 1.0500000000000001e-06,
+      "loss": 4.1276,
+      "step": 21
+    },
+    {
+      "epoch": 0.20121951219512196,
+      "grad_norm": 26.542314529418945,
+      "learning_rate": 1.1e-06,
+      "loss": 4.0641,
+      "step": 22
+    },
+    {
+      "epoch": 0.21036585365853658,
+      "grad_norm": 23.26603126525879,
+      "learning_rate": 1.1500000000000002e-06,
+      "loss": 4.0197,
+      "step": 23
+    },
+    {
+      "epoch": 0.21951219512195122,
+      "grad_norm": 21.04385757446289,
+      "learning_rate": 1.2000000000000002e-06,
+      "loss": 4.0179,
+      "step": 24
+    },
+    {
+      "epoch": 0.22865853658536586,
+      "grad_norm": 19.790103912353516,
+      "learning_rate": 1.25e-06,
+      "loss": 3.8334,
+      "step": 25
+    },
+    {
+      "epoch": 0.23780487804878048,
+      "grad_norm": 18.2396183013916,
+      "learning_rate": 1.3e-06,
+      "loss": 3.7027,
+      "step": 26
+    },
+    {
+      "epoch": 0.24695121951219512,
+      "grad_norm": 18.725860595703125,
+      "learning_rate": 1.3500000000000002e-06,
+      "loss": 3.7268,
+      "step": 27
+    },
+    {
+      "epoch": 0.25609756097560976,
+      "grad_norm": 18.91561508178711,
+      "learning_rate": 1.4000000000000001e-06,
+      "loss": 3.52,
+      "step": 28
+    },
+    {
+      "epoch": 0.2652439024390244,
+      "grad_norm": 18.196651458740234,
+      "learning_rate": 1.45e-06,
+      "loss": 3.4765,
+      "step": 29
+    },
+    {
+      "epoch": 0.27439024390243905,
+      "grad_norm": 17.778606414794922,
+      "learning_rate": 1.5e-06,
+      "loss": 3.3573,
+      "step": 30
+    },
+    {
+      "epoch": 0.28353658536585363,
+      "grad_norm": 16.23012924194336,
+      "learning_rate": 1.5500000000000002e-06,
+      "loss": 3.2783,
+      "step": 31
+    },
+    {
+      "epoch": 0.2926829268292683,
+      "grad_norm": 15.836565017700195,
+      "learning_rate": 1.6000000000000001e-06,
+      "loss": 3.2319,
+      "step": 32
+    },
+    {
+      "epoch": 0.3018292682926829,
+      "grad_norm": 11.919289588928223,
+      "learning_rate": 1.6500000000000003e-06,
+      "loss": 3.1345,
+      "step": 33
+    },
+    {
+      "epoch": 0.31097560975609756,
+      "grad_norm": 11.086843490600586,
+      "learning_rate": 1.7000000000000002e-06,
+      "loss": 3.1153,
+      "step": 34
+    },
+    {
+      "epoch": 0.3201219512195122,
+      "grad_norm": 7.563486576080322,
+      "learning_rate": 1.75e-06,
+      "loss": 2.9784,
+      "step": 35
+    },
+    {
+      "epoch": 0.32926829268292684,
+      "grad_norm": 7.60545539855957,
+      "learning_rate": 1.8000000000000001e-06,
+      "loss": 2.8568,
+      "step": 36
+    },
+    {
+      "epoch": 0.3384146341463415,
+      "grad_norm": 7.071218013763428,
+      "learning_rate": 1.85e-06,
+      "loss": 2.8879,
+      "step": 37
+    },
+    {
+      "epoch": 0.3475609756097561,
+      "grad_norm": 8.200672149658203,
+      "learning_rate": 1.9000000000000002e-06,
+      "loss": 2.7437,
+      "step": 38
+    },
+    {
+      "epoch": 0.3567073170731707,
+      "grad_norm": 12.079374313354492,
+      "learning_rate": 1.9500000000000004e-06,
+      "loss": 2.7589,
+      "step": 39
+    },
+    {
+      "epoch": 0.36585365853658536,
+      "grad_norm": 12.21938705444336,
+      "learning_rate": 2.0000000000000003e-06,
+      "loss": 2.7098,
+      "step": 40
+    },
+    {
+      "epoch": 0.375,
+      "grad_norm": 10.224228858947754,
+      "learning_rate": 2.05e-06,
+      "loss": 2.6803,
+      "step": 41
+    },
+    {
+      "epoch": 0.38414634146341464,
+      "grad_norm": 10.054616928100586,
+      "learning_rate": 2.1000000000000002e-06,
+      "loss": 2.4658,
+      "step": 42
+    },
+    {
+      "epoch": 0.3932926829268293,
+      "grad_norm": 8.074049949645996,
+      "learning_rate": 2.15e-06,
+      "loss": 2.6341,
+      "step": 43
+    },
+    {
+      "epoch": 0.4024390243902439,
+      "grad_norm": 5.721987724304199,
+      "learning_rate": 2.2e-06,
+      "loss": 2.3848,
+      "step": 44
+    },
+    {
+      "epoch": 0.4115853658536585,
+      "grad_norm": 4.646566390991211,
+      "learning_rate": 2.25e-06,
+      "loss": 2.4323,
+      "step": 45
+    },
+    {
+      "epoch": 0.42073170731707316,
+      "grad_norm": 4.591104984283447,
+      "learning_rate": 2.3000000000000004e-06,
+      "loss": 2.4022,
+      "step": 46
+    },
+    {
+      "epoch": 0.4298780487804878,
+      "grad_norm": 4.822715759277344,
+      "learning_rate": 2.35e-06,
+      "loss": 2.2768,
+      "step": 47
+    },
+    {
+      "epoch": 0.43902439024390244,
+      "grad_norm": 5.176568984985352,
+      "learning_rate": 2.4000000000000003e-06,
+      "loss": 2.281,
+      "step": 48
+    },
+    {
+      "epoch": 0.4481707317073171,
+      "grad_norm": 4.731242656707764,
+      "learning_rate": 2.4500000000000003e-06,
+      "loss": 2.235,
+      "step": 49
+    },
+    {
+      "epoch": 0.4573170731707317,
+      "grad_norm": 4.6343865394592285,
+      "learning_rate": 2.5e-06,
+      "loss": 2.1465,
+      "step": 50
+    },
+    {
+      "epoch": 0.46646341463414637,
+      "grad_norm": 4.669382095336914,
+      "learning_rate": 2.55e-06,
+      "loss": 2.0969,
+      "step": 51
+    },
+    {
+      "epoch": 0.47560975609756095,
+      "grad_norm": 4.207769870758057,
+      "learning_rate": 2.6e-06,
+      "loss": 2.0308,
+      "step": 52
+    },
+    {
+      "epoch": 0.4847560975609756,
+      "grad_norm": 5.222470760345459,
+      "learning_rate": 2.6500000000000005e-06,
+      "loss": 1.9445,
+      "step": 53
+    },
+    {
+      "epoch": 0.49390243902439024,
+      "grad_norm": 5.10925817489624,
+      "learning_rate": 2.7000000000000004e-06,
+      "loss": 1.7763,
+      "step": 54
+    },
+    {
+      "epoch": 0.5030487804878049,
+      "grad_norm": 4.8151044845581055,
+      "learning_rate": 2.7500000000000004e-06,
+      "loss": 1.7661,
+      "step": 55
+    },
+    {
+      "epoch": 0.5121951219512195,
+      "grad_norm": 6.155700206756592,
+      "learning_rate": 2.8000000000000003e-06,
+      "loss": 1.6665,
+      "step": 56
+    },
+    {
+      "epoch": 0.5213414634146342,
+      "grad_norm": 4.719021320343018,
+      "learning_rate": 2.85e-06,
+      "loss": 1.5532,
+      "step": 57
+    },
+    {
+      "epoch": 0.5304878048780488,
+      "grad_norm": 4.591799259185791,
+      "learning_rate": 2.9e-06,
+      "loss": 1.5244,
+      "step": 58
+    },
+    {
+      "epoch": 0.5396341463414634,
+      "grad_norm": 5.483795642852783,
+      "learning_rate": 2.95e-06,
+      "loss": 1.3816,
+      "step": 59
+    },
+    {
+      "epoch": 0.5487804878048781,
+      "grad_norm": 4.32553768157959,
+      "learning_rate": 3e-06,
+      "loss": 1.4268,
+      "step": 60
+    },
+    {
+      "epoch": 0.5579268292682927,
+      "grad_norm": 8.296231269836426,
+      "learning_rate": 3.05e-06,
+      "loss": 1.2887,
+      "step": 61
+    },
+    {
+      "epoch": 0.5670731707317073,
+      "grad_norm": 5.07594108581543,
+      "learning_rate": 3.1000000000000004e-06,
+      "loss": 1.228,
+      "step": 62
+    },
+    {
+      "epoch": 0.5762195121951219,
+      "grad_norm": 8.427435874938965,
+      "learning_rate": 3.1500000000000003e-06,
+      "loss": 1.1724,
+      "step": 63
+    },
+    {
+      "epoch": 0.5853658536585366,
+      "grad_norm": 22.45092010498047,
+      "learning_rate": 3.2000000000000003e-06,
+      "loss": 1.0871,
+      "step": 64
+    },
+    {
+      "epoch": 0.5945121951219512,
+      "grad_norm": 20.67412567138672,
+      "learning_rate": 3.2500000000000002e-06,
+      "loss": 1.0108,
+      "step": 65
+    },
+    {
+      "epoch": 0.6036585365853658,
+      "grad_norm": 4.921634674072266,
+      "learning_rate": 3.3000000000000006e-06,
+      "loss": 0.9775,
+      "step": 66
+    },
+    {
+      "epoch": 0.6128048780487805,
+      "grad_norm": 4.477286338806152,
+      "learning_rate": 3.3500000000000005e-06,
+      "loss": 0.9554,
+      "step": 67
+    },
+    {
+      "epoch": 0.6219512195121951,
+      "grad_norm": 8.190194129943848,
+      "learning_rate": 3.4000000000000005e-06,
+      "loss": 0.8583,
+      "step": 68
+    },
+    {
+      "epoch": 0.6310975609756098,
+      "grad_norm": 29.21201515197754,
+      "learning_rate": 3.45e-06,
+      "loss": 0.8499,
+      "step": 69
+    },
+    {
+      "epoch": 0.6402439024390244,
+      "grad_norm": 33.81047439575195,
+      "learning_rate": 3.5e-06,
+      "loss": 0.858,
+      "step": 70
+    },
+    {
+      "epoch": 0.649390243902439,
+      "grad_norm": 3.5125606060028076,
+      "learning_rate": 3.5500000000000003e-06,
+      "loss": 0.7227,
+      "step": 71
+    },
+    {
+      "epoch": 0.6585365853658537,
+      "grad_norm": 18.493423461914062,
+      "learning_rate": 3.6000000000000003e-06,
+      "loss": 0.7447,
+      "step": 72
+    },
+    {
+      "epoch": 0.6676829268292683,
+      "grad_norm": 24.923927307128906,
+      "learning_rate": 3.65e-06,
+      "loss": 0.8423,
+      "step": 73
+    },
+    {
+      "epoch": 0.676829268292683,
+      "grad_norm": 9.870631217956543,
+      "learning_rate": 3.7e-06,
+      "loss": 0.6934,
+      "step": 74
+    },
+    {
+      "epoch": 0.6859756097560976,
+      "grad_norm": 20.598569869995117,
+      "learning_rate": 3.7500000000000005e-06,
+      "loss": 0.7005,
+      "step": 75
+    },
+    {
+      "epoch": 0.6951219512195121,
+      "grad_norm": 30.662466049194336,
+      "learning_rate": 3.8000000000000005e-06,
+      "loss": 0.6226,
+      "step": 76
+    },
+    {
+      "epoch": 0.7042682926829268,
+      "grad_norm": 28.930774688720703,
+      "learning_rate": 3.85e-06,
+      "loss": 0.6357,
+      "step": 77
+    },
+    {
+      "epoch": 0.7134146341463414,
+      "grad_norm": 7.140467166900635,
+      "learning_rate": 3.900000000000001e-06,
+      "loss": 0.6129,
+      "step": 78
+    },
+    {
+      "epoch": 0.7225609756097561,
+      "grad_norm": 6.322911739349365,
+      "learning_rate": 3.95e-06,
+      "loss": 0.6086,
+      "step": 79
+    },
+    {
+      "epoch": 0.7317073170731707,
+      "grad_norm": 5.404141426086426,
+      "learning_rate": 4.000000000000001e-06,
+      "loss": 0.5986,
+      "step": 80
+    },
+    {
+      "epoch": 0.7408536585365854,
+      "grad_norm": 24.445228576660156,
+      "learning_rate": 4.05e-06,
+      "loss": 0.5932,
+      "step": 81
+    },
+    {
+      "epoch": 0.75,
+      "grad_norm": 14.275381088256836,
+      "learning_rate": 4.1e-06,
+      "loss": 0.5286,
+      "step": 82
+    },
+    {
+      "epoch": 0.7591463414634146,
+      "grad_norm": 9.77132511138916,
+      "learning_rate": 4.15e-06,
+      "loss": 0.4814,
+      "step": 83
+    },
+    {
+      "epoch": 0.7682926829268293,
+      "grad_norm": 11.043930053710938,
+      "learning_rate": 4.2000000000000004e-06,
+      "loss": 0.4953,
+      "step": 84
+    },
+    {
+      "epoch": 0.7774390243902439,
+      "grad_norm": 3.1333775520324707,
+      "learning_rate": 4.25e-06,
+      "loss": 0.4794,
+      "step": 85
+    },
+    {
+      "epoch": 0.7865853658536586,
+      "grad_norm": 9.50448989868164,
+      "learning_rate": 4.3e-06,
+      "loss": 0.4451,
+      "step": 86
+    },
+    {
+      "epoch": 0.7957317073170732,
+      "grad_norm": 5.867074489593506,
+      "learning_rate": 4.350000000000001e-06,
+      "loss": 0.4224,
+      "step": 87
+    },
+    {
+      "epoch": 0.8048780487804879,
+      "grad_norm": 12.556106567382812,
+      "learning_rate": 4.4e-06,
+      "loss": 0.4224,
+      "step": 88
+    },
+    {
+      "epoch": 0.8140243902439024,
+      "grad_norm": 7.916622638702393,
+      "learning_rate": 4.450000000000001e-06,
+      "loss": 0.4091,
+      "step": 89
+    },
+    {
+      "epoch": 0.823170731707317,
+      "grad_norm": 2.970932722091675,
+      "learning_rate": 4.5e-06,
+      "loss": 0.4181,
+      "step": 90
+    },
+    {
+      "epoch": 0.8323170731707317,
+      "grad_norm": 17.468870162963867,
+      "learning_rate": 4.5500000000000005e-06,
+      "loss": 0.4031,
+      "step": 91
+    },
+    {
+      "epoch": 0.8414634146341463,
+      "grad_norm": 15.861083984375,
+      "learning_rate": 4.600000000000001e-06,
+      "loss": 0.3604,
+      "step": 92
+    },
+    {
+      "epoch": 0.850609756097561,
+      "grad_norm": 6.715958118438721,
+      "learning_rate": 4.65e-06,
+      "loss": 0.3489,
+      "step": 93
+    },
+    {
+      "epoch": 0.8597560975609756,
+      "grad_norm": 5.439844608306885,
+      "learning_rate": 4.7e-06,
+      "loss": 0.3339,
+      "step": 94
+    },
+    {
+      "epoch": 0.8689024390243902,
+      "grad_norm": 4.650434494018555,
+      "learning_rate": 4.75e-06,
+      "loss": 0.3189,
+      "step": 95
+    },
+    {
+      "epoch": 0.8780487804878049,
+      "grad_norm": 15.319528579711914,
+      "learning_rate": 4.800000000000001e-06,
+      "loss": 0.3347,
+      "step": 96
+    },
+    {
+      "epoch": 0.8871951219512195,
+      "grad_norm": 5.151925563812256,
+      "learning_rate": 4.85e-06,
+      "loss": 0.3043,
+      "step": 97
+    },
+    {
+      "epoch": 0.8963414634146342,
+      "grad_norm": 6.351178169250488,
+      "learning_rate": 4.9000000000000005e-06,
+      "loss": 0.2939,
+      "step": 98
+    },
+    {
+      "epoch": 0.9054878048780488,
+      "grad_norm": 3.942911148071289,
+      "learning_rate": 4.95e-06,
+      "loss": 0.2887,
+      "step": 99
+    },
+    {
+      "epoch": 0.9146341463414634,
+      "grad_norm": 3.0885493755340576,
+      "learning_rate": 5e-06,
+      "loss": 0.2992,
+      "step": 100
+    },
+    {
+      "epoch": 0.9237804878048781,
+      "grad_norm": 7.023274898529053,
+      "learning_rate": 4.999959803423605e-06,
+      "loss": 0.2607,
+      "step": 101
+    },
+    {
+      "epoch": 0.9329268292682927,
+      "grad_norm": 8.774216651916504,
+      "learning_rate": 4.9998392149870315e-06,
+      "loss": 0.2674,
+      "step": 102
+    },
+    {
+      "epoch": 0.9420731707317073,
+      "grad_norm": 3.5439791679382324,
+      "learning_rate": 4.999638238568072e-06,
+      "loss": 0.2602,
+      "step": 103
+    },
+    {
+      "epoch": 0.9512195121951219,
+      "grad_norm": 7.669381618499756,
+      "learning_rate": 4.999356880629579e-06,
+      "loss": 0.2718,
+      "step": 104
+    },
+    {
+      "epoch": 0.9603658536585366,
+      "grad_norm": 6.9407854080200195,
+      "learning_rate": 4.998995150219252e-06,
+      "loss": 0.2494,
+      "step": 105
+    },
+    {
+      "epoch": 0.9695121951219512,
+      "grad_norm": 12.144411087036133,
+      "learning_rate": 4.998553058969352e-06,
+      "loss": 0.2297,
+      "step": 106
+    },
+    {
+      "epoch": 0.9786585365853658,
+      "grad_norm": 6.956138610839844,
+      "learning_rate": 4.998030621096322e-06,
+      "loss": 0.2386,
+      "step": 107
+    },
+    {
+      "epoch": 0.9878048780487805,
+      "grad_norm": 8.304037094116211,
+      "learning_rate": 4.997427853400333e-06,
+      "loss": 0.23,
+      "step": 108
+    },
+    {
+      "epoch": 0.9969512195121951,
+      "grad_norm": 7.031365394592285,
+      "learning_rate": 4.996744775264743e-06,
+      "loss": 0.2346,
+      "step": 109
+    },
+    {
+      "epoch": 1.0,
+      "grad_norm": 7.031365394592285,
+      "learning_rate": 4.9959814086554735e-06,
+      "loss": 0.0739,
+      "step": 110
+    },
+    {
+      "epoch": 1.0091463414634145,
+      "grad_norm": 2.084608793258667,
+      "learning_rate": 4.995137778120306e-06,
+      "loss": 0.2422,
+      "step": 111
+    },
+    {
+      "epoch": 1.0182926829268293,
+      "grad_norm": 38.12245178222656,
+      "learning_rate": 4.994213910788085e-06,
+      "loss": 0.1992,
+      "step": 112
+    },
+    {
+      "epoch": 1.0274390243902438,
+      "grad_norm": 2.817565441131592,
+      "learning_rate": 4.993209836367856e-06,
+      "loss": 0.1882,
+      "step": 113
+    },
+    {
+      "epoch": 1.0365853658536586,
+      "grad_norm": 11.774249076843262,
+      "learning_rate": 4.9921255871479e-06,
+      "loss": 0.2235,
+      "step": 114
+    },
+    {
+      "epoch": 1.045731707317073,
+      "grad_norm": 2.3362019062042236,
+      "learning_rate": 4.9909611979947046e-06,
+      "loss": 0.1875,
+      "step": 115
+    },
+    {
+      "epoch": 1.0548780487804879,
+      "grad_norm": 1.9516818523406982,
+      "learning_rate": 4.989716706351835e-06,
+      "loss": 0.1858,
+      "step": 116
+    },
+    {
+      "epoch": 1.0640243902439024,
+      "grad_norm": 1.5484645366668701,
+      "learning_rate": 4.9883921522387336e-06,
+      "loss": 0.1845,
+      "step": 117
+    },
+    {
+      "epoch": 1.0731707317073171,
+      "grad_norm": 7.430396556854248,
+      "learning_rate": 4.986987578249433e-06,
+      "loss": 0.184,
+      "step": 118
+    },
+    {
+      "epoch": 1.0823170731707317,
+      "grad_norm": 2.0725948810577393,
+      "learning_rate": 4.9855030295511845e-06,
+      "loss": 0.177,
+      "step": 119
+    },
+    {
+      "epoch": 1.0914634146341464,
+      "grad_norm": 4.836520671844482,
+      "learning_rate": 4.98393855388301e-06,
+      "loss": 0.1744,
+      "step": 120
+    },
+    {
+      "epoch": 1.100609756097561,
+      "grad_norm": 5.22581672668457,
+      "learning_rate": 4.982294201554162e-06,
+      "loss": 0.1599,
+      "step": 121
+    },
+    {
+      "epoch": 1.1097560975609757,
+      "grad_norm": 5.406208038330078,
+      "learning_rate": 4.980570025442507e-06,
+      "loss": 0.1645,
+      "step": 122
+    },
+    {
+      "epoch": 1.1189024390243902,
+      "grad_norm": 8.736640930175781,
+      "learning_rate": 4.978766080992827e-06,
+      "loss": 0.1943,
+      "step": 123
+    },
+    {
+      "epoch": 1.1280487804878048,
+      "grad_norm": 9.184198379516602,
+      "learning_rate": 4.976882426215035e-06,
+      "loss": 0.1753,
+      "step": 124
+    },
+    {
+      "epoch": 1.1371951219512195,
+      "grad_norm": 5.317800521850586,
+      "learning_rate": 4.974919121682307e-06,
+      "loss": 0.1897,
+      "step": 125
+    },
+    {
+      "epoch": 1.146341463414634,
+      "grad_norm": 11.803364753723145,
+      "learning_rate": 4.972876230529143e-06,
+      "loss": 0.1516,
+      "step": 126
+    },
+    {
+      "epoch": 1.1554878048780488,
+      "grad_norm": 31.74310302734375,
+      "learning_rate": 4.9707538184493255e-06,
+      "loss": 0.1947,
+      "step": 127
+    },
+    {
+      "epoch": 1.1646341463414633,
+      "grad_norm": 1.956932783126831,
+      "learning_rate": 4.968551953693813e-06,
+      "loss": 0.1708,
+      "step": 128
+    },
+    {
+      "epoch": 1.173780487804878,
+      "grad_norm": 4.64445686340332,
+      "learning_rate": 4.966270707068548e-06,
+      "loss": 0.1711,
+      "step": 129
+    },
+    {
+      "epoch": 1.1829268292682926,
+      "grad_norm": 4.442819595336914,
+      "learning_rate": 4.963910151932171e-06,
+      "loss": 0.1572,
+      "step": 130
+    },
+    {
+      "epoch": 1.1920731707317074,
+      "grad_norm": 1.9702759981155396,
+      "learning_rate": 4.961470364193673e-06,
+      "loss": 0.1364,
+      "step": 131
+    },
+    {
+      "epoch": 1.201219512195122,
+      "grad_norm": 7.816587448120117,
+      "learning_rate": 4.9589514223099425e-06,
+      "loss": 0.1624,
+      "step": 132
+    },
+    {
+      "epoch": 1.2103658536585367,
+      "grad_norm": 61.18424606323242,
+      "learning_rate": 4.956353407283252e-06,
+      "loss": 0.26,
+      "step": 133
+    },
+    {
+      "epoch": 1.2195121951219512,
+      "grad_norm": 15.649847984313965,
+      "learning_rate": 4.9536764026586505e-06,
+      "loss": 0.1543,
+      "step": 134
+    },
+    {
+      "epoch": 1.228658536585366,
+      "grad_norm": 2.249363660812378,
+      "learning_rate": 4.950920494521274e-06,
+      "loss": 0.152,
+      "step": 135
+    },
+    {
+      "epoch": 1.2378048780487805,
+      "grad_norm": 4.055661678314209,
+      "learning_rate": 4.948085771493579e-06,
+      "loss": 0.1402,
+      "step": 136
+    },
+    {
+      "epoch": 1.2469512195121952,
+      "grad_norm": 2.2803292274475098,
+      "learning_rate": 4.945172324732496e-06,
+      "loss": 0.1427,
+      "step": 137
+    },
+    {
+      "epoch": 1.2560975609756098,
+      "grad_norm": 2.5540456771850586,
+      "learning_rate": 4.942180247926492e-06,
+      "loss": 0.143,
+      "step": 138
+    },
+    {
+      "epoch": 1.2652439024390243,
+      "grad_norm": 8.80340576171875,
+      "learning_rate": 4.939109637292563e-06,
+      "loss": 0.1569,
+      "step": 139
+    },
+    {
+      "epoch": 1.274390243902439,
+      "grad_norm": 3.1077523231506348,
+      "learning_rate": 4.935960591573136e-06,
+      "loss": 0.1313,
+      "step": 140
+    },
+    {
+      "epoch": 1.2835365853658536,
+      "grad_norm": 1.2506815195083618,
+      "learning_rate": 4.932733212032896e-06,
+      "loss": 0.137,
+      "step": 141
+    },
+    {
+      "epoch": 1.2926829268292683,
+      "grad_norm": 2.4397666454315186,
+      "learning_rate": 4.929427602455532e-06,
+      "loss": 0.1408,
+      "step": 142
+    },
+    {
+      "epoch": 1.3018292682926829,
+      "grad_norm": 1.7658663988113403,
+      "learning_rate": 4.926043869140392e-06,
+      "loss": 0.133,
+      "step": 143
+    },
+    {
+      "epoch": 1.3109756097560976,
+      "grad_norm": 1.0473108291625977,
+      "learning_rate": 4.922582120899072e-06,
+      "loss": 0.1358,
+      "step": 144
+    },
+    {
+      "epoch": 1.3201219512195121,
+      "grad_norm": 5.566607475280762,
+      "learning_rate": 4.919042469051916e-06,
+      "loss": 0.1237,
+      "step": 145
+    },
+    {
+      "epoch": 1.329268292682927,
+      "grad_norm": 1.0360139608383179,
+      "learning_rate": 4.915425027424429e-06,
+      "loss": 0.1274,
+      "step": 146
+    },
+    {
+      "epoch": 1.3384146341463414,
+      "grad_norm": 1.9091176986694336,
+      "learning_rate": 4.911729912343631e-06,
+      "loss": 0.1439,
+      "step": 147
+    },
+    {
+      "epoch": 1.3475609756097562,
+      "grad_norm": 1.5727638006210327,
+      "learning_rate": 4.907957242634299e-06,
+      "loss": 0.1259,
+      "step": 148
+    },
+    {
+      "epoch": 1.3567073170731707,
+      "grad_norm": 1.0574792623519897,
+      "learning_rate": 4.9041071396151585e-06,
+      "loss": 0.142,
+      "step": 149
+    },
+    {
+      "epoch": 1.3658536585365852,
+      "grad_norm": 8.020130157470703,
+      "learning_rate": 4.900179727094978e-06,
+      "loss": 0.1392,
+      "step": 150
+    },
+    {
+      "epoch": 1.375,
+      "grad_norm": 3.156255006790161,
+      "learning_rate": 4.896175131368587e-06,
+      "loss": 0.152,
+      "step": 151
+    },
+    {
+      "epoch": 1.3841463414634148,
+      "grad_norm": 2.3956990242004395,
+      "learning_rate": 4.892093481212817e-06,
+      "loss": 0.1244,
+      "step": 152
+    },
+    {
+      "epoch": 1.3932926829268293,
+      "grad_norm": 2.6058578491210938,
+      "learning_rate": 4.887934907882357e-06,
+      "loss": 0.1271,
+      "step": 153
+    },
+    {
+      "epoch": 1.4024390243902438,
+      "grad_norm": 1.7173289060592651,
+      "learning_rate": 4.883699545105535e-06,
+      "loss": 0.1257,
+      "step": 154
+    },
+    {
+      "epoch": 1.4115853658536586,
+      "grad_norm": 2.2237367630004883,
+      "learning_rate": 4.8793875290800186e-06,
+      "loss": 0.127,
+      "step": 155
+    },
+    {
+      "epoch": 1.420731707317073,
+      "grad_norm": 6.997623920440674,
+      "learning_rate": 4.874998998468433e-06,
+      "loss": 0.1454,
+      "step": 156
+    },
+    {
+      "epoch": 1.4298780487804879,
+      "grad_norm": 10.645139694213867,
+      "learning_rate": 4.870534094393903e-06,
+      "loss": 0.1402,
+      "step": 157
+    },
+    {
+      "epoch": 1.4390243902439024,
+      "grad_norm": 1.5808072090148926,
+      "learning_rate": 4.865992960435514e-06,
+      "loss": 0.1156,
+      "step": 158
+    },
+    {
+      "epoch": 1.4481707317073171,
+      "grad_norm": 0.990803062915802,
+      "learning_rate": 4.861375742623697e-06,
+      "loss": 0.1286,
+      "step": 159
+    },
+    {
+      "epoch": 1.4573170731707317,
+      "grad_norm": 1.5008662939071655,
+      "learning_rate": 4.856682589435531e-06,
+      "loss": 0.1282,
+      "step": 160
+    },
+    {
+      "epoch": 1.4664634146341464,
+      "grad_norm": 1.772304892539978,
+      "learning_rate": 4.851913651789968e-06,
+      "loss": 0.1308,
+      "step": 161
+    },
+    {
+      "epoch": 1.475609756097561,
+      "grad_norm": 1.201061487197876,
+      "learning_rate": 4.847069083042982e-06,
+      "loss": 0.1198,
+      "step": 162
+    },
+    {
+      "epoch": 1.4847560975609757,
+      "grad_norm": 0.6147828698158264,
+      "learning_rate": 4.8421490389826344e-06,
+      "loss": 0.1138,
+      "step": 163
+    },
+    {
+      "epoch": 1.4939024390243902,
+      "grad_norm": 1.3990256786346436,
+      "learning_rate": 4.837153677824068e-06,
+      "loss": 0.1205,
+      "step": 164
+    },
+    {
+      "epoch": 1.5030487804878048,
+      "grad_norm": 1.6977574825286865,
+      "learning_rate": 4.832083160204414e-06,
+      "loss": 0.1128,
+      "step": 165
+    },
+    {
+      "epoch": 1.5121951219512195,
+      "grad_norm": 2.4056453704833984,
+      "learning_rate": 4.826937649177632e-06,
+      "loss": 0.1184,
+      "step": 166
+    },
+    {
+      "epoch": 1.5213414634146343,
+      "grad_norm": 2.7455694675445557,
+      "learning_rate": 4.821717310209265e-06,
+      "loss": 0.123,
+      "step": 167
+    },
+    {
+      "epoch": 1.5304878048780488,
+      "grad_norm": 1.8243584632873535,
+      "learning_rate": 4.816422311171115e-06,
+      "loss": 0.1198,
+      "step": 168
+    },
+    {
+      "epoch": 1.5396341463414633,
+      "grad_norm": 0.8920577764511108,
+      "learning_rate": 4.811052822335849e-06,
+      "loss": 0.1088,
+      "step": 169
+    },
+    {
+      "epoch": 1.548780487804878,
+      "grad_norm": 1.6639117002487183,
+      "learning_rate": 4.805609016371522e-06,
+      "loss": 0.1231,
+      "step": 170
+    },
+    {
+      "epoch": 1.5579268292682928,
+      "grad_norm": 1.383026123046875,
+      "learning_rate": 4.800091068336024e-06,
+      "loss": 0.1207,
+      "step": 171
+    },
+    {
+      "epoch": 1.5670731707317072,
+      "grad_norm": 1.6599416732788086,
+      "learning_rate": 4.79449915567145e-06,
+      "loss": 0.1206,
+      "step": 172
+    },
+    {
+      "epoch": 1.576219512195122,
+      "grad_norm": 1.5868676900863647,
+      "learning_rate": 4.788833458198395e-06,
+      "loss": 0.1158,
+      "step": 173
+    },
+    {
+      "epoch": 1.5853658536585367,
+      "grad_norm": 1.0154211521148682,
+      "learning_rate": 4.783094158110175e-06,
+      "loss": 0.1091,
+      "step": 174
+    },
+    {
+      "epoch": 1.5945121951219512,
+      "grad_norm": 2.1514394283294678,
+      "learning_rate": 4.7772814399669585e-06,
+      "loss": 0.1109,
+      "step": 175
+    },
+    {
+      "epoch": 1.6036585365853657,
+      "grad_norm": 0.7475085258483887,
+      "learning_rate": 4.7713954906898415e-06,
+      "loss": 0.1162,
+      "step": 176
+    },
+    {
+      "epoch": 1.6128048780487805,
+      "grad_norm": 0.8523163795471191,
+      "learning_rate": 4.7654364995548325e-06,
+      "loss": 0.1238,
+      "step": 177
+    },
+    {
+      "epoch": 1.6219512195121952,
+      "grad_norm": 1.5456233024597168,
+      "learning_rate": 4.759404658186765e-06,
+      "loss": 0.1163,
+      "step": 178
+    },
+    {
+      "epoch": 1.6310975609756098,
+      "grad_norm": 1.123340129852295,
+      "learning_rate": 4.753300160553137e-06,
+      "loss": 0.1104,
+      "step": 179
+    },
+    {
+      "epoch": 1.6402439024390243,
+      "grad_norm": 1.16841721534729,
+      "learning_rate": 4.747123202957871e-06,
+      "loss": 0.1099,
+      "step": 180
+    },
+    {
+      "epoch": 1.649390243902439,
+      "grad_norm": 2.025221109390259,
+      "learning_rate": 4.740873984035009e-06,
+      "loss": 0.1157,
+      "step": 181
+    },
+    {
+      "epoch": 1.6585365853658538,
+      "grad_norm": 3.876469373703003,
+      "learning_rate": 4.734552704742314e-06,
+      "loss": 0.1122,
+      "step": 182
+    },
+    {
+      "epoch": 1.6676829268292683,
+      "grad_norm": 1.5562145709991455,
+      "learning_rate": 4.728159568354814e-06,
+      "loss": 0.1127,
+      "step": 183
+    },
+    {
+      "epoch": 1.6768292682926829,
+      "grad_norm": 1.4691755771636963,
+      "learning_rate": 4.721694780458266e-06,
+      "loss": 0.1182,
+      "step": 184
+    },
+    {
+      "epoch": 1.6859756097560976,
+      "grad_norm": 1.9529973268508911,
+      "learning_rate": 4.715158548942541e-06,
+      "loss": 0.1153,
+      "step": 185
+    },
+    {
+      "epoch": 1.6951219512195121,
+      "grad_norm": 1.9065771102905273,
+      "learning_rate": 4.708551083994944e-06,
+      "loss": 0.1163,
+      "step": 186
+    },
+    {
+      "epoch": 1.7042682926829267,
+      "grad_norm": 1.0964735746383667,
+      "learning_rate": 4.701872598093452e-06,
+      "loss": 0.114,
+      "step": 187
+    },
+    {
+      "epoch": 1.7134146341463414,
+      "grad_norm": 1.0885926485061646,
+      "learning_rate": 4.695123305999877e-06,
+      "loss": 0.114,
+      "step": 188
+    },
+    {
+      "epoch": 1.7225609756097562,
+      "grad_norm": 1.1777565479278564,
+      "learning_rate": 4.688303424752969e-06,
+      "loss": 0.1116,
+      "step": 189
+    },
+    {
+      "epoch": 1.7317073170731707,
+      "grad_norm": 1.3686976432800293,
+      "learning_rate": 4.6814131736614286e-06,
+      "loss": 0.1081,
+      "step": 190
+    },
+    {
+      "epoch": 1.7408536585365852,
+      "grad_norm": 0.8220391869544983,
+      "learning_rate": 4.674452774296861e-06,
+      "loss": 0.1091,
+      "step": 191
+    },
+    {
+      "epoch": 1.75,
+      "grad_norm": 1.073935866355896,
+      "learning_rate": 4.667422450486646e-06,
+      "loss": 0.1148,
+      "step": 192
+    },
+    {
+      "epoch": 1.7591463414634148,
+      "grad_norm": 1.389614224433899,
+      "learning_rate": 4.660322428306741e-06,
+      "loss": 0.1134,
+      "step": 193
+    },
+    {
+      "epoch": 1.7682926829268293,
+      "grad_norm": 1.349023699760437,
+      "learning_rate": 4.653152936074413e-06,
+      "loss": 0.1056,
+      "step": 194
+    },
+    {
+      "epoch": 1.7774390243902438,
+      "grad_norm": 1.0334906578063965,
+      "learning_rate": 4.645914204340898e-06,
+      "loss": 0.113,
+      "step": 195
+    },
+    {
+      "epoch": 1.7865853658536586,
+      "grad_norm": 18.6312313079834,
+      "learning_rate": 4.638606465883979e-06,
+      "loss": 0.1351,
+      "step": 196
+    },
+    {
+      "epoch": 1.7957317073170733,
+      "grad_norm": 0.7669345736503601,
+      "learning_rate": 4.6312299557005125e-06,
+      "loss": 0.1097,
+      "step": 197
+    },
+    {
+      "epoch": 1.8048780487804879,
+      "grad_norm": 2.1091127395629883,
+      "learning_rate": 4.623784910998862e-06,
+      "loss": 0.1122,
+      "step": 198
+    },
+    {
+      "epoch": 1.8140243902439024,
+      "grad_norm": 0.8031914830207825,
+      "learning_rate": 4.616271571191273e-06,
+      "loss": 0.1109,
+      "step": 199
+    },
+    {
+      "epoch": 1.8231707317073171,
+      "grad_norm": 2.1154661178588867,
+      "learning_rate": 4.608690177886177e-06,
+      "loss": 0.1131,
+      "step": 200
+    },
+    {
+      "epoch": 1.8323170731707317,
+      "grad_norm": 1.4946709871292114,
+      "learning_rate": 4.6010409748804165e-06,
+      "loss": 0.1136,
+      "step": 201
+    },
+    {
+      "epoch": 1.8414634146341462,
+      "grad_norm": 1.0509642362594604,
+      "learning_rate": 4.593324208151412e-06,
+      "loss": 0.1162,
+      "step": 202
+    },
+    {
+      "epoch": 1.850609756097561,
+      "grad_norm": 0.8742370009422302,
+      "learning_rate": 4.585540125849244e-06,
+      "loss": 0.1102,
+      "step": 203
+    },
+    {
+      "epoch": 1.8597560975609757,
+      "grad_norm": 1.1283056735992432,
+      "learning_rate": 4.577688978288681e-06,
+      "loss": 0.1104,
+      "step": 204
+    },
+    {
+      "epoch": 1.8689024390243902,
+      "grad_norm": 1.0081048011779785,
+      "learning_rate": 4.569771017941125e-06,
+      "loss": 0.1119,
+      "step": 205
+    },
+    {
+      "epoch": 1.8780487804878048,
+      "grad_norm": 0.7291656732559204,
+      "learning_rate": 4.561786499426494e-06,
+      "loss": 0.1103,
+      "step": 206
+    },
+    {
+      "epoch": 1.8871951219512195,
+      "grad_norm": 1.144096851348877,
+      "learning_rate": 4.5537356795050346e-06,
+      "loss": 0.1125,
+      "step": 207
+    },
+    {
+      "epoch": 1.8963414634146343,
+      "grad_norm": 0.7317456603050232,
+      "learning_rate": 4.545618817069066e-06,
+      "loss": 0.1105,
+      "step": 208
+    },
+    {
+      "epoch": 1.9054878048780488,
+      "grad_norm": 1.1636486053466797,
+      "learning_rate": 4.537436173134653e-06,
+      "loss": 0.1091,
+      "step": 209
+    },
+    {
+      "epoch": 1.9146341463414633,
+      "grad_norm": 0.6319037675857544,
+      "learning_rate": 4.529188010833212e-06,
+      "loss": 0.1088,
+      "step": 210
+    },
+    {
+      "epoch": 1.923780487804878,
+      "grad_norm": 0.7535589933395386,
+      "learning_rate": 4.520874595403053e-06,
+      "loss": 0.1041,
+      "step": 211
+    },
+    {
+      "epoch": 1.9329268292682928,
+      "grad_norm": 1.0461041927337646,
+      "learning_rate": 4.512496194180846e-06,
+      "loss": 0.1129,
+      "step": 212
+    },
+    {
+      "epoch": 1.9420731707317072,
+      "grad_norm": 0.6124874949455261,
+      "learning_rate": 4.5040530765930275e-06,
+      "loss": 0.1113,
+      "step": 213
+    },
+    {
+      "epoch": 1.951219512195122,
+      "grad_norm": 0.8516119122505188,
+      "learning_rate": 4.495545514147134e-06,
+      "loss": 0.1112,
+      "step": 214
+    },
+    {
+      "epoch": 1.9603658536585367,
+      "grad_norm": 0.8919075131416321,
+      "learning_rate": 4.486973780423073e-06,
+      "loss": 0.1089,
+      "step": 215
+    },
+    {
+      "epoch": 1.9695121951219512,
+      "grad_norm": 1.2183221578598022,
+      "learning_rate": 4.478338151064323e-06,
+      "loss": 0.1156,
+      "step": 216
+    },
+    {
+      "epoch": 1.9786585365853657,
+      "grad_norm": 1.0123646259307861,
+      "learning_rate": 4.4696389037690734e-06,
+      "loss": 0.1077,
+      "step": 217
+    },
+    {
+      "epoch": 1.9878048780487805,
+      "grad_norm": 0.8304896950721741,
+      "learning_rate": 4.460876318281291e-06,
+      "loss": 0.1051,
+      "step": 218
+    },
+    {
+      "epoch": 1.9969512195121952,
+      "grad_norm": 0.5879438519477844,
+      "learning_rate": 4.452050676381725e-06,
+      "loss": 0.1064,
+      "step": 219
+    },
+    {
+      "epoch": 2.0,
+      "grad_norm": 0.5879438519477844,
+      "learning_rate": 4.443162261878846e-06,
+      "loss": 0.0344,
+      "step": 220
+    },
+    {
+      "epoch": 2.0091463414634148,
+      "grad_norm": 0.8598087430000305,
+      "learning_rate": 4.434211360599721e-06,
+      "loss": 0.1071,
+      "step": 221
+    },
+    {
+      "epoch": 2.018292682926829,
+      "grad_norm": 0.5539699196815491,
+      "learning_rate": 4.425198260380818e-06,
+      "loss": 0.1095,
+      "step": 222
+    },
+    {
+      "epoch": 2.027439024390244,
+      "grad_norm": 0.4581831395626068,
+      "learning_rate": 4.416123251058756e-06,
+      "loss": 0.104,
+      "step": 223
+    },
+    {
+      "epoch": 2.0365853658536586,
+      "grad_norm": 0.550529420375824,
+      "learning_rate": 4.406986624460979e-06,
+      "loss": 0.1049,
+      "step": 224
+    },
+    {
+      "epoch": 2.0457317073170733,
+      "grad_norm": 0.749291181564331,
+      "learning_rate": 4.397788674396374e-06,
+      "loss": 0.1091,
+      "step": 225
+    },
+    {
+      "epoch": 2.0548780487804876,
+      "grad_norm": 0.7612536549568176,
+      "learning_rate": 4.3885296966458225e-06,
+      "loss": 0.1045,
+      "step": 226
+    },
+    {
+      "epoch": 2.0640243902439024,
+      "grad_norm": 0.6671517491340637,
+      "learning_rate": 4.37920998895269e-06,
+      "loss": 0.1057,
+      "step": 227
+    },
+    {
+      "epoch": 2.073170731707317,
+      "grad_norm": 0.5838347673416138,
+      "learning_rate": 4.369829851013251e-06,
+      "loss": 0.1061,
+      "step": 228
+    },
+    {
+      "epoch": 2.082317073170732,
+      "grad_norm": 1.937151312828064,
+      "learning_rate": 4.36038958446705e-06,
+      "loss": 0.1074,
+      "step": 229
+    },
+    {
+      "epoch": 2.091463414634146,
+      "grad_norm": 0.8083934187889099,
+      "learning_rate": 4.350889492887202e-06,
+      "loss": 0.1096,
+      "step": 230
+    },
+    {
+      "epoch": 2.100609756097561,
+      "grad_norm": 0.6810253262519836,
+      "learning_rate": 4.341329881770635e-06,
+      "loss": 0.1024,
+      "step": 231
+    },
+    {
+      "epoch": 2.1097560975609757,
+      "grad_norm": 0.8901084065437317,
+      "learning_rate": 4.331711058528259e-06,
+      "loss": 0.1064,
+      "step": 232
+    },
+    {
+      "epoch": 2.1189024390243905,
+      "grad_norm": 0.34419459104537964,
+      "learning_rate": 4.322033332475084e-06,
+      "loss": 0.1074,
+      "step": 233
+    },
+    {
+      "epoch": 2.1280487804878048,
+      "grad_norm": 0.8596321940422058,
+      "learning_rate": 4.312297014820275e-06,
+      "loss": 0.1061,
+      "step": 234
+    },
+    {
+      "epoch": 2.1371951219512195,
+      "grad_norm": 0.8391798138618469,
+      "learning_rate": 4.302502418657138e-06,
+      "loss": 0.107,
+      "step": 235
+    },
+    {
+      "epoch": 2.1463414634146343,
+      "grad_norm": 0.7193841338157654,
+      "learning_rate": 4.292649858953063e-06,
+      "loss": 0.1083,
+      "step": 236
+    },
+    {
+      "epoch": 2.1554878048780486,
+      "grad_norm": 1.5975165367126465,
+      "learning_rate": 4.282739652539384e-06,
+      "loss": 0.109,
+      "step": 237
+    },
+    {
+      "epoch": 2.1646341463414633,
+      "grad_norm": 0.9933533072471619,
+      "learning_rate": 4.272772118101195e-06,
+      "loss": 0.1073,
+      "step": 238
+    },
+    {
+      "epoch": 2.173780487804878,
+      "grad_norm": 0.9281901717185974,
+      "learning_rate": 4.262747576167106e-06,
+      "loss": 0.1066,
+      "step": 239
+    },
+    {
+      "epoch": 2.182926829268293,
+      "grad_norm": 3.2005884647369385,
+      "learning_rate": 4.2526663490989264e-06,
+      "loss": 0.1106,
+      "step": 240
+    },
+    {
+      "epoch": 2.192073170731707,
+      "grad_norm": 1.084905982017517,
+      "learning_rate": 4.24252876108131e-06,
+      "loss": 0.1047,
+      "step": 241
+    },
+    {
+      "epoch": 2.201219512195122,
+      "grad_norm": 0.5345124006271362,
+      "learning_rate": 4.232335138111321e-06,
+      "loss": 0.1027,
+      "step": 242
+    },
+    {
+      "epoch": 2.2103658536585367,
+      "grad_norm": 0.6235584020614624,
+      "learning_rate": 4.222085807987954e-06,
+      "loss": 0.1032,
+      "step": 243
+    },
+    {
+      "epoch": 2.2195121951219514,
+      "grad_norm": 0.9797384738922119,
+      "learning_rate": 4.211781100301596e-06,
+      "loss": 0.1082,
+      "step": 244
+    },
+    {
+      "epoch": 2.2286585365853657,
+      "grad_norm": 0.3665165901184082,
+      "learning_rate": 4.201421346423421e-06,
+      "loss": 0.1057,
+      "step": 245
+    },
+    {
+      "epoch": 2.2378048780487805,
+      "grad_norm": 1.232487440109253,
+      "learning_rate": 4.19100687949474e-06,
+      "loss": 0.1088,
+      "step": 246
+    },
+    {
+      "epoch": 2.2469512195121952,
+      "grad_norm": 0.8028481602668762,
+      "learning_rate": 4.180538034416287e-06,
+      "loss": 0.1119,
+      "step": 247
+    },
+    {
+      "epoch": 2.2560975609756095,
+      "grad_norm": 1.094025731086731,
+      "learning_rate": 4.1700151478374436e-06,
+      "loss": 0.1033,
+      "step": 248
+    },
+    {
+      "epoch": 2.2652439024390243,
+      "grad_norm": 1.0119608640670776,
+      "learning_rate": 4.159438558145425e-06,
+      "loss": 0.108,
+      "step": 249
+    },
+    {
+      "epoch": 2.274390243902439,
+      "grad_norm": 0.6864210963249207,
+      "learning_rate": 4.148808605454385e-06,
+      "loss": 0.1072,
+      "step": 250
+    },
+    {
+      "epoch": 2.283536585365854,
+      "grad_norm": 0.8415966629981995,
+      "learning_rate": 4.138125631594488e-06,
+      "loss": 0.1052,
+      "step": 251
+    },
+    {
+      "epoch": 2.292682926829268,
+      "grad_norm": 0.7150280475616455,
+      "learning_rate": 4.127389980100916e-06,
+      "loss": 0.1049,
+      "step": 252
+    },
+    {
+      "epoch": 2.301829268292683,
+      "grad_norm": 1.2521971464157104,
+      "learning_rate": 4.116601996202815e-06,
+      "loss": 0.1078,
+      "step": 253
+    },
+    {
+      "epoch": 2.3109756097560976,
+      "grad_norm": 0.8204450011253357,
+      "learning_rate": 4.1057620268122e-06,
+      "loss": 0.1078,
+      "step": 254
+    },
+    {
+      "epoch": 2.3201219512195124,
+      "grad_norm": 1.0992320775985718,
+      "learning_rate": 4.0948704205128e-06,
+      "loss": 0.1063,
+      "step": 255
+    },
+    {
+      "epoch": 2.3292682926829267,
+      "grad_norm": 0.4928065240383148,
+      "learning_rate": 4.08392752754884e-06,
+      "loss": 0.1076,
+      "step": 256
+    },
+    {
+      "epoch": 2.3384146341463414,
+      "grad_norm": 1.1588791608810425,
+      "learning_rate": 4.072933699813788e-06,
+      "loss": 0.1075,
+      "step": 257
+    },
+    {
+      "epoch": 2.347560975609756,
+      "grad_norm": 0.5810270309448242,
+      "learning_rate": 4.061889290839032e-06,
+      "loss": 0.107,
+      "step": 258
+    },
+    {
+      "epoch": 2.3567073170731705,
+      "grad_norm": 0.8428746461868286,
+      "learning_rate": 4.050794655782515e-06,
+      "loss": 0.1057,
+      "step": 259
+    },
+    {
+      "epoch": 2.3658536585365852,
+      "grad_norm": 1.4898464679718018,
+      "learning_rate": 4.039650151417316e-06,
+      "loss": 0.1043,
+      "step": 260
+    },
+    {
+      "epoch": 2.375,
+      "grad_norm": 0.8613987565040588,
+      "learning_rate": 4.028456136120169e-06,
+      "loss": 0.1011,
+      "step": 261
+    },
+    {
+      "epoch": 2.3841463414634148,
+      "grad_norm": 0.7771743535995483,
+      "learning_rate": 4.017212969859948e-06,
+      "loss": 0.1073,
+      "step": 262
+    },
+    {
+      "epoch": 2.3932926829268295,
+      "grad_norm": 0.4884893596172333,
+      "learning_rate": 4.0059210141860875e-06,
+      "loss": 0.104,
+      "step": 263
+    },
+    {
+      "epoch": 2.402439024390244,
+      "grad_norm": 0.8188875317573547,
+      "learning_rate": 3.994580632216952e-06,
+      "loss": 0.1043,
+      "step": 264
+    },
+    {
+      "epoch": 2.4115853658536586,
+      "grad_norm": 0.814239501953125,
+      "learning_rate": 3.9831921886281675e-06,
+      "loss": 0.1073,
+      "step": 265
+    },
+    {
+      "epoch": 2.4207317073170733,
+      "grad_norm": 1.029391884803772,
+      "learning_rate": 3.971756049640888e-06,
+      "loss": 0.103,
+      "step": 266
+    },
+    {
+      "epoch": 2.4298780487804876,
+      "grad_norm": 0.3826993405818939,
+      "learning_rate": 3.96027258301002e-06,
+      "loss": 0.1089,
+      "step": 267
+    },
+    {
+      "epoch": 2.4390243902439024,
+      "grad_norm": 0.5306946039199829,
+      "learning_rate": 3.9487421580123995e-06,
+      "loss": 0.1086,
+      "step": 268
+    },
+    {
+      "epoch": 2.448170731707317,
+      "grad_norm": 1.286549687385559,
+      "learning_rate": 3.937165145434914e-06,
+      "loss": 0.1085,
+      "step": 269
+    },
+    {
+      "epoch": 2.457317073170732,
+      "grad_norm": 1.5009950399398804,
+      "learning_rate": 3.92554191756258e-06,
+      "loss": 0.1068,
+      "step": 270
+    },
+    {
+      "epoch": 2.466463414634146,
+      "grad_norm": 0.9225317239761353,
+      "learning_rate": 3.9138728481665696e-06,
+      "loss": 0.1059,
+      "step": 271
+    },
+    {
+      "epoch": 2.475609756097561,
+      "grad_norm": 0.5062177181243896,
+      "learning_rate": 3.902158312492196e-06,
+      "loss": 0.1089,
+      "step": 272
+    },
+    {
+      "epoch": 2.4847560975609757,
+      "grad_norm": 0.6272971630096436,
+      "learning_rate": 3.8903986872468416e-06,
+      "loss": 0.1099,
+      "step": 273
+    },
+    {
+      "epoch": 2.4939024390243905,
+      "grad_norm": 1.207027554512024,
+      "learning_rate": 3.878594350587844e-06,
+      "loss": 0.1086,
+      "step": 274
+    },
+    {
+      "epoch": 2.5030487804878048,
+      "grad_norm": 0.8417999744415283,
+      "learning_rate": 3.866745682110343e-06,
+      "loss": 0.1091,
+      "step": 275
+    },
+    {
+      "epoch": 2.5121951219512195,
+      "grad_norm": 0.5926170945167542,
+      "learning_rate": 3.854853062835062e-06,
+      "loss": 0.1081,
+      "step": 276
+    },
+    {
+      "epoch": 2.5213414634146343,
+      "grad_norm": 0.5806312561035156,
+      "learning_rate": 3.842916875196066e-06,
+      "loss": 0.1057,
+      "step": 277
+    },
+    {
+      "epoch": 2.5304878048780486,
+      "grad_norm": 1.2453746795654297,
+      "learning_rate": 3.830937503028457e-06,
+      "loss": 0.1086,
+      "step": 278
+    },
+    {
+      "epoch": 2.5396341463414633,
+      "grad_norm": 0.32152408361434937,
+      "learning_rate": 3.818915331556032e-06,
+      "loss": 0.1076,
+      "step": 279
+    },
+    {
+      "epoch": 2.548780487804878,
+      "grad_norm": 0.5756719708442688,
+      "learning_rate": 3.8068507473789016e-06,
+      "loss": 0.1057,
+      "step": 280
+    },
+    {
+      "epoch": 2.557926829268293,
+      "grad_norm": 0.9345139861106873,
+      "learning_rate": 3.7947441384610473e-06,
+      "loss": 0.1043,
+      "step": 281
+    },
+    {
+      "epoch": 2.567073170731707,
+      "grad_norm": 0.6475419998168945,
+      "learning_rate": 3.7825958941178538e-06,
+      "loss": 0.1046,
+      "step": 282
+    },
+    {
+      "epoch": 2.576219512195122,
+      "grad_norm": 0.7345425486564636,
+      "learning_rate": 3.770406405003586e-06,
+      "loss": 0.1042,
+      "step": 283
+    },
+    {
+      "epoch": 2.5853658536585367,
+      "grad_norm": 0.769500195980072,
+      "learning_rate": 3.758176063098829e-06,
+      "loss": 0.1048,
+      "step": 284
+    },
+    {
+      "epoch": 2.5945121951219514,
+      "grad_norm": 0.5934444665908813,
+      "learning_rate": 3.745905261697881e-06,
+      "loss": 0.1086,
+      "step": 285
+    },
+    {
+      "epoch": 2.6036585365853657,
+      "grad_norm": 0.5718421936035156,
+      "learning_rate": 3.7335943953961063e-06,
+      "loss": 0.1027,
+      "step": 286
+    },
+    {
+      "epoch": 2.6128048780487805,
+      "grad_norm": 0.6114512085914612,
+      "learning_rate": 3.721243860077247e-06,
+      "loss": 0.1091,
+      "step": 287
+    },
+    {
+      "epoch": 2.6219512195121952,
+      "grad_norm": 0.5620512366294861,
+      "learning_rate": 3.708854052900693e-06,
+      "loss": 0.1088,
+      "step": 288
+    },
+    {
+      "epoch": 2.6310975609756095,
+      "grad_norm": 0.6152999997138977,
+      "learning_rate": 3.696425372288708e-06,
+      "loss": 0.1035,
+      "step": 289
+    },
+    {
+      "epoch": 2.6402439024390243,
+      "grad_norm": 0.9731057286262512,
+      "learning_rate": 3.68395821791362e-06,
+      "loss": 0.1066,
+      "step": 290
+    },
+    {
+      "epoch": 2.649390243902439,
+      "grad_norm": 0.49733713269233704,
+      "learning_rate": 3.6714529906849683e-06,
+      "loss": 0.1045,
+      "step": 291
+    },
+    {
+      "epoch": 2.658536585365854,
+      "grad_norm": 0.49706941843032837,
+      "learning_rate": 3.65891009273661e-06,
+      "loss": 0.1035,
+      "step": 292
+    },
+    {
+      "epoch": 2.6676829268292686,
+      "grad_norm": 0.32593467831611633,
+      "learning_rate": 3.6463299274137883e-06,
+      "loss": 0.1017,
+      "step": 293
+    },
+    {
+      "epoch": 2.676829268292683,
+      "grad_norm": 0.4569077491760254,
+      "learning_rate": 3.6337128992601665e-06,
+      "loss": 0.1061,
+      "step": 294
+    },
+    {
+      "epoch": 2.6859756097560976,
+      "grad_norm": 0.5540208220481873,
+      "learning_rate": 3.621059414004811e-06,
+      "loss": 0.106,
+      "step": 295
+    },
+    {
+      "epoch": 2.6951219512195124,
+      "grad_norm": 0.9319402575492859,
+      "learning_rate": 3.6083698785491537e-06,
+      "loss": 0.1042,
+      "step": 296
+    },
+    {
+      "epoch": 2.7042682926829267,
+      "grad_norm": 0.795711874961853,
+      "learning_rate": 3.595644700953898e-06,
+      "loss": 0.104,
+      "step": 297
+    },
+    {
+      "epoch": 2.7134146341463414,
+      "grad_norm": 0.4707932472229004,
+      "learning_rate": 3.582884290425903e-06,
+      "loss": 0.1087,
+      "step": 298
+    },
+    {
+      "epoch": 2.722560975609756,
+      "grad_norm": 0.7709797620773315,
+      "learning_rate": 3.5700890573050224e-06,
+      "loss": 0.107,
+      "step": 299
+    },
+    {
+      "epoch": 2.7317073170731705,
+      "grad_norm": 1.1342726945877075,
+      "learning_rate": 3.5572594130509074e-06,
+      "loss": 0.1029,
+      "step": 300
+    },
+    {
+      "epoch": 2.7408536585365852,
+      "grad_norm": 0.9055543541908264,
+      "learning_rate": 3.544395770229781e-06,
+      "loss": 0.1087,
+      "step": 301
+    },
+    {
+      "epoch": 2.75,
+      "grad_norm": 1.084656000137329,
+      "learning_rate": 3.531498542501161e-06,
+      "loss": 0.1044,
+      "step": 302
+    },
+    {
+      "epoch": 2.7591463414634148,
+      "grad_norm": 0.5991970300674438,
+      "learning_rate": 3.51856814460457e-06,
+      "loss": 0.1064,
+      "step": 303
+    },
+    {
+      "epoch": 2.7682926829268295,
+      "grad_norm": 1.7364325523376465,
+      "learning_rate": 3.5056049923461873e-06,
+      "loss": 0.1079,
+      "step": 304
+    },
+    {
+      "epoch": 2.777439024390244,
+      "grad_norm": 0.4037036895751953,
+      "learning_rate": 3.4926095025854863e-06,
+      "loss": 0.1058,
+      "step": 305
+    },
+    {
+      "epoch": 2.7865853658536586,
+      "grad_norm": 0.8944678902626038,
+      "learning_rate": 3.479582093221824e-06,
+      "loss": 0.1064,
+      "step": 306
+    },
+    {
+      "epoch": 2.7957317073170733,
+      "grad_norm": 0.5730510950088501,
+      "learning_rate": 3.4665231831810055e-06,
+      "loss": 0.1055,
+      "step": 307
+    },
+    {
+      "epoch": 2.8048780487804876,
+      "grad_norm": 0.519359827041626,
+      "learning_rate": 3.45343319240181e-06,
+      "loss": 0.1069,
+      "step": 308
+    },
+    {
+      "epoch": 2.8140243902439024,
+      "grad_norm": 0.6074658632278442,
+      "learning_rate": 3.44031254182249e-06,
+      "loss": 0.1058,
+      "step": 309
+    },
+    {
+      "epoch": 2.823170731707317,
+      "grad_norm": 0.37584108114242554,
+      "learning_rate": 3.4271616533672312e-06,
+      "loss": 0.1052,
+      "step": 310
+    },
+    {
+      "epoch": 2.8323170731707314,
+      "grad_norm": 0.7730855941772461,
+      "learning_rate": 3.413980949932589e-06,
+      "loss": 0.1037,
+      "step": 311
+    },
+    {
+      "epoch": 2.841463414634146,
+      "grad_norm": 0.9912458062171936,
+      "learning_rate": 3.4007708553738837e-06,
+      "loss": 0.1039,
+      "step": 312
+    },
+    {
+      "epoch": 2.850609756097561,
+      "grad_norm": 0.36861947178840637,
+      "learning_rate": 3.3875317944915765e-06,
+      "loss": 0.1083,
+      "step": 313
+    },
+    {
+      "epoch": 2.8597560975609757,
+      "grad_norm": 0.6069959402084351,
+      "learning_rate": 3.3742641930176044e-06,
+      "loss": 0.1063,
+      "step": 314
+    },
+    {
+      "epoch": 2.8689024390243905,
+      "grad_norm": 0.5478480458259583,
+      "learning_rate": 3.3609684776016932e-06,
+      "loss": 0.1033,
+      "step": 315
+    },
+    {
+      "epoch": 2.8780487804878048,
+      "grad_norm": 0.8352198004722595,
+      "learning_rate": 3.3476450757976353e-06,
+      "loss": 0.1026,
+      "step": 316
+    },
+    {
+      "epoch": 2.8871951219512195,
+      "grad_norm": 0.32641175389289856,
+      "learning_rate": 3.3342944160495405e-06,
+      "loss": 0.1065,
+      "step": 317
+    },
+    {
+      "epoch": 2.8963414634146343,
+      "grad_norm": 0.8076167702674866,
+      "learning_rate": 3.3209169276780612e-06,
+      "loss": 0.105,
+      "step": 318
+    },
+    {
+      "epoch": 2.9054878048780486,
+      "grad_norm": 0.3602111041545868,
+      "learning_rate": 3.307513040866584e-06,
+      "loss": 0.106,
+      "step": 319
+    },
+    {
+      "epoch": 2.9146341463414633,
+      "grad_norm": 0.227371945977211,
+      "learning_rate": 3.2940831866473967e-06,
+      "loss": 0.1029,
+      "step": 320
+    },
+    {
+      "epoch": 2.923780487804878,
+      "grad_norm": 0.5158879160881042,
+      "learning_rate": 3.280627796887828e-06,
+      "loss": 0.1046,
+      "step": 321
+    },
+    {
+      "epoch": 2.932926829268293,
+      "grad_norm": 1.0091784000396729,
+      "learning_rate": 3.2671473042763603e-06,
+      "loss": 0.1063,
+      "step": 322
+    },
+    {
+      "epoch": 2.942073170731707,
+      "grad_norm": 0.9310367107391357,
+      "learning_rate": 3.2536421423087137e-06,
+      "loss": 0.1015,
+      "step": 323
+    },
+    {
+      "epoch": 2.951219512195122,
+      "grad_norm": 0.7602329254150391,
+      "learning_rate": 3.2401127452739106e-06,
+      "loss": 0.1061,
+      "step": 324
+    },
+    {
+      "epoch": 2.9603658536585367,
+      "grad_norm": 0.3374696671962738,
+      "learning_rate": 3.2265595482403002e-06,
+      "loss": 0.1049,
+      "step": 325
+    },
+    {
+      "epoch": 2.9695121951219514,
+      "grad_norm": 0.8469128012657166,
+      "learning_rate": 3.212982987041582e-06,
+      "loss": 0.1059,
+      "step": 326
+    },
+    {
+      "epoch": 2.9786585365853657,
+      "grad_norm": 0.7546894550323486,
+      "learning_rate": 3.199383498262777e-06,
+      "loss": 0.1067,
+      "step": 327
+    },
+    {
+      "epoch": 2.9878048780487805,
+      "grad_norm": 0.471538245677948,
+      "learning_rate": 3.185761519226199e-06,
+      "loss": 0.1054,
+      "step": 328
+    },
+    {
+      "epoch": 2.9969512195121952,
+      "grad_norm": 1.0999424457550049,
+      "learning_rate": 3.1721174879773825e-06,
+      "loss": 0.1067,
+      "step": 329
+    },
+    {
+      "epoch": 3.0,
+      "grad_norm": 0.9866787791252136,
+      "learning_rate": 3.1584518432710055e-06,
+      "loss": 0.0361,
+      "step": 330
+    },
+    {
+      "epoch": 3.0091463414634148,
+      "grad_norm": 0.4876601994037628,
+      "learning_rate": 3.1447650245567714e-06,
+      "loss": 0.1058,
+      "step": 331
+    },
+    {
+      "epoch": 3.018292682926829,
+      "grad_norm": 1.1444916725158691,
+      "learning_rate": 3.131057471965283e-06,
+      "loss": 0.1051,
+      "step": 332
+    },
+    {
+      "epoch": 3.027439024390244,
+      "grad_norm": 0.6264286637306213,
+      "learning_rate": 3.117329626293891e-06,
+      "loss": 0.1065,
+      "step": 333
+    },
+    {
+      "epoch": 3.0365853658536586,
+      "grad_norm": 0.49345099925994873,
+      "learning_rate": 3.10358192899251e-06,
+      "loss": 0.1034,
+      "step": 334
+    },
+    {
+      "epoch": 3.0457317073170733,
+      "grad_norm": 0.663642168045044,
+      "learning_rate": 3.089814822149435e-06,
+      "loss": 0.1063,
+      "step": 335
+    },
+    {
+      "epoch": 3.0548780487804876,
+      "grad_norm": 0.7982928156852722,
+      "learning_rate": 3.0760287484771124e-06,
+      "loss": 0.1047,
+      "step": 336
+    },
+    {
+      "epoch": 3.0640243902439024,
+      "grad_norm": 0.7954766750335693,
+      "learning_rate": 3.062224151297915e-06,
+      "loss": 0.1069,
+      "step": 337
+    },
+    {
+      "epoch": 3.073170731707317,
+      "grad_norm": 0.42658311128616333,
+      "learning_rate": 3.0484014745298784e-06,
+      "loss": 0.1043,
+      "step": 338
+    },
+    {
+      "epoch": 3.082317073170732,
+      "grad_norm": 0.8154664635658264,
+      "learning_rate": 3.0345611626724286e-06,
+      "loss": 0.1061,
+      "step": 339
+    },
+    {
+      "epoch": 3.091463414634146,
+      "grad_norm": 0.6405104994773865,
+      "learning_rate": 3.020703660792089e-06,
+      "loss": 0.1044,
+      "step": 340
+    },
+    {
+      "epoch": 3.100609756097561,
+      "grad_norm": 0.7365081906318665,
+      "learning_rate": 3.006829414508164e-06,
+      "loss": 0.1061,
+      "step": 341
+    },
+    {
+      "epoch": 3.1097560975609757,
+      "grad_norm": 0.322336345911026,
+      "learning_rate": 2.9929388699784163e-06,
+      "loss": 0.1067,
+      "step": 342
+    },
+    {
+      "epoch": 3.1189024390243905,
+      "grad_norm": 1.0929139852523804,
+      "learning_rate": 2.9790324738847117e-06,
+      "loss": 0.1084,
+      "step": 343
+    },
+    {
+      "epoch": 3.1280487804878048,
+      "grad_norm": 0.9065106511116028,
+      "learning_rate": 2.965110673418662e-06,
+      "loss": 0.1029,
+      "step": 344
+    },
+    {
+      "epoch": 3.1371951219512195,
+      "grad_norm": 0.7359282970428467,
+      "learning_rate": 2.9511739162672377e-06,
+      "loss": 0.1022,
+      "step": 345
+    },
+    {
+      "epoch": 3.1463414634146343,
+      "grad_norm": 0.5595942139625549,
+      "learning_rate": 2.9372226505983803e-06,
+      "loss": 0.1067,
+      "step": 346
+    },
+    {
+      "epoch": 3.1554878048780486,
+      "grad_norm": 0.9696919918060303,
+      "learning_rate": 2.923257325046581e-06,
+      "loss": 0.1085,
+      "step": 347
+    },
+    {
+      "epoch": 3.1646341463414633,
+      "grad_norm": 0.3462243676185608,
+      "learning_rate": 2.9092783886984615e-06,
+      "loss": 0.1022,
+      "step": 348
+    },
+    {
+      "epoch": 3.173780487804878,
+      "grad_norm": 0.5291770696640015,
+      "learning_rate": 2.895286291078327e-06,
+      "loss": 0.1053,
+      "step": 349
+    },
+    {
+      "epoch": 3.182926829268293,
+      "grad_norm": 0.667945921421051,
+      "learning_rate": 2.881281482133715e-06,
+      "loss": 0.1031,
+      "step": 350
+    },
+    {
+      "epoch": 3.192073170731707,
+      "grad_norm": 1.4323768615722656,
+      "learning_rate": 2.8672644122209224e-06,
+      "loss": 0.1081,
+      "step": 351
+    },
+    {
+      "epoch": 3.201219512195122,
+      "grad_norm": 0.6841598749160767,
+      "learning_rate": 2.8532355320905277e-06,
+      "loss": 0.106,
+      "step": 352
+    },
+    {
+      "epoch": 3.2103658536585367,
+      "grad_norm": 1.017216444015503,
+      "learning_rate": 2.839195292872892e-06,
+      "loss": 0.1087,
+      "step": 353
+    },
+    {
+      "epoch": 3.2195121951219514,
+      "grad_norm": 0.6531134843826294,
+      "learning_rate": 2.8251441460636542e-06,
+      "loss": 0.1046,
+      "step": 354
+    },
+    {
+      "epoch": 3.2286585365853657,
+      "grad_norm": 0.5456640124320984,
+      "learning_rate": 2.8110825435092108e-06,
+      "loss": 0.1039,
+      "step": 355
+    },
+    {
+      "epoch": 3.2378048780487805,
+      "grad_norm": 0.7548927068710327,
+      "learning_rate": 2.797010937392188e-06,
+      "loss": 0.1031,
+      "step": 356
+    },
+    {
+      "epoch": 3.2469512195121952,
+      "grad_norm": 0.3874664008617401,
+      "learning_rate": 2.782929780216896e-06,
+      "loss": 0.1044,
+      "step": 357
+    },
+    {
+      "epoch": 3.2560975609756095,
+      "grad_norm": 0.41182073950767517,
+      "learning_rate": 2.768839524794784e-06,
+      "loss": 0.1041,
+      "step": 358
+    },
+    {
+      "epoch": 3.2652439024390243,
+      "grad_norm": 0.9750642776489258,
+      "learning_rate": 2.754740624229875e-06,
+      "loss": 0.1074,
+      "step": 359
+    },
+    {
+      "epoch": 3.274390243902439,
+      "grad_norm": 0.5468454957008362,
+      "learning_rate": 2.740633531904196e-06,
+      "loss": 0.1056,
+      "step": 360
+    },
+    {
+      "epoch": 3.283536585365854,
+      "grad_norm": 0.6265747547149658,
+      "learning_rate": 2.7265187014631983e-06,
+      "loss": 0.0993,
+      "step": 361
+    },
+    {
+      "epoch": 3.292682926829268,
+      "grad_norm": 0.5447110533714294,
+      "learning_rate": 2.7123965868011693e-06,
+      "loss": 0.1023,
+      "step": 362
+    },
+    {
+      "epoch": 3.301829268292683,
+      "grad_norm": 0.5873153805732727,
+      "learning_rate": 2.698267642046639e-06,
+      "loss": 0.1043,
+      "step": 363
+    },
+    {
+      "epoch": 3.3109756097560976,
+      "grad_norm": 0.8410884141921997,
+      "learning_rate": 2.6841323215477716e-06,
+      "loss": 0.1026,
+      "step": 364
+    },
+    {
+      "epoch": 3.3201219512195124,
+      "grad_norm": 0.6379916667938232,
+      "learning_rate": 2.6699910798577612e-06,
+      "loss": 0.1043,
+      "step": 365
+    },
+    {
+      "epoch": 3.3292682926829267,
+      "grad_norm": 0.3435944616794586,
+      "learning_rate": 2.6558443717202077e-06,
+      "loss": 0.105,
+      "step": 366
+    },
+    {
+      "epoch": 3.3384146341463414,
+      "grad_norm": 1.0160895586013794,
+      "learning_rate": 2.6416926520545e-06,
+      "loss": 0.1029,
+      "step": 367
+    },
+    {
+      "epoch": 3.347560975609756,
+      "grad_norm": 0.6623978018760681,
+      "learning_rate": 2.6275363759411817e-06,
+      "loss": 0.1023,
+      "step": 368
+    },
+    {
+      "epoch": 3.3567073170731705,
+      "grad_norm": 0.42660218477249146,
+      "learning_rate": 2.61337599860732e-06,
+      "loss": 0.102,
+      "step": 369
+    },
+    {
+      "epoch": 3.3658536585365852,
+      "grad_norm": 0.992641270160675,
+      "learning_rate": 2.599211975411867e-06,
+      "loss": 0.1087,
+      "step": 370
+    },
+    {
+      "epoch": 3.375,
+      "grad_norm": 0.9266735911369324,
+      "learning_rate": 2.585044761831014e-06,
+      "loss": 0.102,
+      "step": 371
+    },
+    {
+      "epoch": 3.3841463414634148,
+      "grad_norm": 0.7853158712387085,
+      "learning_rate": 2.5708748134435503e-06,
+      "loss": 0.1062,
+      "step": 372
+    },
+    {
+      "epoch": 3.3932926829268295,
+      "grad_norm": 0.501575767993927,
+      "learning_rate": 2.556702585916202e-06,
+      "loss": 0.1034,
+      "step": 373
+    },
+    {
+      "epoch": 3.402439024390244,
+      "grad_norm": 0.4489349126815796,
+      "learning_rate": 2.5425285349889937e-06,
+      "loss": 0.1072,
+      "step": 374
+    },
+    {
+      "epoch": 3.4115853658536586,
+      "grad_norm": 0.6775074601173401,
+      "learning_rate": 2.52835311646058e-06,
+      "loss": 0.1069,
+      "step": 375
+    },
+    {
+      "epoch": 3.4207317073170733,
+      "grad_norm": 0.6220641732215881,
+      "learning_rate": 2.5141767861735976e-06,
+      "loss": 0.1057,
+      "step": 376
+    },
+    {
+      "epoch": 3.4298780487804876,
+      "grad_norm": 0.8748292326927185,
+      "learning_rate": 2.5e-06,
+      "loss": 0.1043,
+      "step": 377
+    },
+    {
+      "epoch": 3.4390243902439024,
+      "grad_norm": 0.9639992117881775,
+      "learning_rate": 2.485823213826403e-06,
+      "loss": 0.1086,
+      "step": 378
+    },
+    {
+      "epoch": 3.448170731707317,
+      "grad_norm": 0.7457119822502136,
+      "learning_rate": 2.47164688353942e-06,
+      "loss": 0.1066,
+      "step": 379
+    },
+    {
+      "epoch": 3.457317073170732,
+      "grad_norm": 0.9384257197380066,
+      "learning_rate": 2.457471465011007e-06,
+      "loss": 0.1045,
+      "step": 380
+    },
+    {
+      "epoch": 3.466463414634146,
+      "grad_norm": 1.3412197828292847,
+      "learning_rate": 2.4432974140837988e-06,
+      "loss": 0.1041,
+      "step": 381
+    },
+    {
+      "epoch": 3.475609756097561,
+      "grad_norm": 0.5068745017051697,
+      "learning_rate": 2.4291251865564505e-06,
+      "loss": 0.1032,
+      "step": 382
+    },
+    {
+      "epoch": 3.4847560975609757,
+      "grad_norm": 0.40219390392303467,
+      "learning_rate": 2.414955238168986e-06,
+      "loss": 0.1039,
+      "step": 383
+    },
+    {
+      "epoch": 3.4939024390243905,
+      "grad_norm": 1.0244395732879639,
+      "learning_rate": 2.4007880245881345e-06,
+      "loss": 0.1049,
+      "step": 384
+    },
+    {
+      "epoch": 3.5030487804878048,
+      "grad_norm": 0.7072446942329407,
+      "learning_rate": 2.386624001392681e-06,
+      "loss": 0.1062,
+      "step": 385
+    },
+    {
+      "epoch": 3.5121951219512195,
+      "grad_norm": 0.7629527449607849,
+      "learning_rate": 2.3724636240588196e-06,
+      "loss": 0.1034,
+      "step": 386
+    },
+    {
+      "epoch": 3.5213414634146343,
+      "grad_norm": 0.8184047937393188,
+      "learning_rate": 2.358307347945501e-06,
+      "loss": 0.1018,
+      "step": 387
+    },
+    {
+      "epoch": 3.5304878048780486,
+      "grad_norm": 0.353141188621521,
+      "learning_rate": 2.3441556282797935e-06,
+      "loss": 0.1077,
+      "step": 388
+    },
+    {
+      "epoch": 3.5396341463414633,
+      "grad_norm": 0.572818398475647,
+      "learning_rate": 2.33000892014224e-06,
+      "loss": 0.1047,
+      "step": 389
+    },
+    {
+      "epoch": 3.548780487804878,
+      "grad_norm": 0.7230311632156372,
+      "learning_rate": 2.315867678452229e-06,
+      "loss": 0.1025,
+      "step": 390
+    },
+    {
+      "epoch": 3.557926829268293,
+      "grad_norm": 1.1688281297683716,
+      "learning_rate": 2.3017323579533617e-06,
+      "loss": 0.1036,
+      "step": 391
+    },
+    {
+      "epoch": 3.567073170731707,
+      "grad_norm": 0.8227838277816772,
+      "learning_rate": 2.287603413198831e-06,
+      "loss": 0.104,
+      "step": 392
+    },
+    {
+      "epoch": 3.576219512195122,
+      "grad_norm": 0.45366042852401733,
+      "learning_rate": 2.2734812985368034e-06,
+      "loss": 0.1042,
+      "step": 393
+    },
+    {
+      "epoch": 3.5853658536585367,
+      "grad_norm": 0.31546878814697266,
+      "learning_rate": 2.2593664680958048e-06,
+      "loss": 0.0998,
+      "step": 394
+    },
+    {
+      "epoch": 3.5945121951219514,
+      "grad_norm": 0.3552347719669342,
+      "learning_rate": 2.2452593757701254e-06,
+      "loss": 0.1032,
+      "step": 395
+    },
+    {
+      "epoch": 3.6036585365853657,
+      "grad_norm": 0.38970789313316345,
+      "learning_rate": 2.231160475205216e-06,
+      "loss": 0.1014,
+      "step": 396
+    },
+    {
+      "epoch": 3.6128048780487805,
+      "grad_norm": 0.32332485914230347,
+      "learning_rate": 2.2170702197831052e-06,
+      "loss": 0.1042,
+      "step": 397
+    },
+    {
+      "epoch": 3.6219512195121952,
+      "grad_norm": 0.5800347924232483,
+      "learning_rate": 2.2029890626078133e-06,
+      "loss": 0.1025,
+      "step": 398
+    },
+    {
+      "epoch": 3.6310975609756095,
+      "grad_norm": 0.5717971920967102,
+      "learning_rate": 2.1889174564907897e-06,
+      "loss": 0.1022,
+      "step": 399
+    },
+    {
+      "epoch": 3.6402439024390243,
+      "grad_norm": 0.4129480719566345,
+      "learning_rate": 2.1748558539363458e-06,
+      "loss": 0.1043,
+      "step": 400
+    },
+    {
+      "epoch": 3.649390243902439,
+      "grad_norm": 0.8331505656242371,
+      "learning_rate": 2.1608047071271087e-06,
+      "loss": 0.103,
+      "step": 401
+    },
+    {
+      "epoch": 3.658536585365854,
+      "grad_norm": 0.7323583364486694,
+      "learning_rate": 2.1467644679094727e-06,
+      "loss": 0.1066,
+      "step": 402
+    },
+    {
+      "epoch": 3.6676829268292686,
+      "grad_norm": 0.828615665435791,
+      "learning_rate": 2.132735587779078e-06,
+      "loss": 0.1029,
+      "step": 403
+    },
+    {
+      "epoch": 3.676829268292683,
+      "grad_norm": 0.35115236043930054,
+      "learning_rate": 2.118718517866286e-06,
+      "loss": 0.1005,
+      "step": 404
+    },
+    {
+      "epoch": 3.6859756097560976,
+      "grad_norm": 0.45387038588523865,
+      "learning_rate": 2.1047137089216728e-06,
+      "loss": 0.1043,
+      "step": 405
+    },
+    {
+      "epoch": 3.6951219512195124,
+      "grad_norm": 0.6801460981369019,
+      "learning_rate": 2.09072161130154e-06,
+      "loss": 0.1013,
+      "step": 406
+    },
+    {
+      "epoch": 3.7042682926829267,
+      "grad_norm": 0.5863631367683411,
+      "learning_rate": 2.0767426749534194e-06,
+      "loss": 0.1008,
+      "step": 407
+    },
+    {
+      "epoch": 3.7134146341463414,
+      "grad_norm": 0.8629023432731628,
+      "learning_rate": 2.0627773494016205e-06,
+      "loss": 0.1058,
+      "step": 408
+    },
+    {
+      "epoch": 3.722560975609756,
+      "grad_norm": 0.929577112197876,
+      "learning_rate": 2.0488260837327627e-06,
+      "loss": 0.1043,
+      "step": 409
+    },
+    {
+      "epoch": 3.7317073170731705,
+      "grad_norm": 0.4356759488582611,
+      "learning_rate": 2.0348893265813397e-06,
+      "loss": 0.1029,
+      "step": 410
+    },
+    {
+      "epoch": 3.7408536585365852,
+      "grad_norm": 1.105850100517273,
+      "learning_rate": 2.0209675261152896e-06,
+      "loss": 0.104,
+      "step": 411
+    },
+    {
+      "epoch": 3.75,
+      "grad_norm": 0.5208716988563538,
+      "learning_rate": 2.0070611300215846e-06,
+      "loss": 0.103,
+      "step": 412
+    },
+    {
+      "epoch": 3.7591463414634148,
+      "grad_norm": 0.6509872674942017,
+      "learning_rate": 1.993170585491836e-06,
+      "loss": 0.1032,
+      "step": 413
+    },
+    {
+      "epoch": 3.7682926829268295,
+      "grad_norm": 0.48762744665145874,
+      "learning_rate": 1.9792963392079125e-06,
+      "loss": 0.1065,
+      "step": 414
+    },
+    {
+      "epoch": 3.777439024390244,
+      "grad_norm": 0.4907792806625366,
+      "learning_rate": 1.9654388373275722e-06,
+      "loss": 0.1018,
+      "step": 415
+    },
+    {
+      "epoch": 3.7865853658536586,
+      "grad_norm": 0.6827414035797119,
+      "learning_rate": 1.951598525470122e-06,
+      "loss": 0.103,
+      "step": 416
+    },
+    {
+      "epoch": 3.7957317073170733,
+      "grad_norm": 0.915044367313385,
+      "learning_rate": 1.937775848702086e-06,
+      "loss": 0.1033,
+      "step": 417
+    },
+    {
+      "epoch": 3.8048780487804876,
+      "grad_norm": 0.7613033056259155,
+      "learning_rate": 1.923971251522888e-06,
+      "loss": 0.1047,
+      "step": 418
+    },
+    {
+      "epoch": 3.8140243902439024,
+      "grad_norm": 0.5619191527366638,
+      "learning_rate": 1.9101851778505666e-06,
+      "loss": 0.1031,
+      "step": 419
+    },
+    {
+      "epoch": 3.823170731707317,
+      "grad_norm": 0.8900383710861206,
+      "learning_rate": 1.8964180710074905e-06,
+      "loss": 0.1059,
+      "step": 420
+    },
+    {
+      "epoch": 3.8323170731707314,
+      "grad_norm": 1.0572234392166138,
+      "learning_rate": 1.8826703737061097e-06,
+      "loss": 0.1062,
+      "step": 421
+    },
+    {
+      "epoch": 3.841463414634146,
+      "grad_norm": 0.379605233669281,
+      "learning_rate": 1.8689425280347168e-06,
+      "loss": 0.1016,
+      "step": 422
+    },
+    {
+      "epoch": 3.850609756097561,
+      "grad_norm": 0.5158970355987549,
+      "learning_rate": 1.8552349754432303e-06,
+      "loss": 0.1044,
+      "step": 423
+    },
+    {
+      "epoch": 3.8597560975609757,
+      "grad_norm": 0.5448965430259705,
+      "learning_rate": 1.8415481567289956e-06,
+      "loss": 0.1055,
+      "step": 424
+    },
+    {
+      "epoch": 3.8689024390243905,
+      "grad_norm": 0.6450077891349792,
+      "learning_rate": 1.8278825120226179e-06,
+      "loss": 0.1015,
+      "step": 425
+    },
+    {
+      "epoch": 3.8780487804878048,
+      "grad_norm": 0.7701898813247681,
+      "learning_rate": 1.8142384807738023e-06,
+      "loss": 0.1019,
+      "step": 426
+    },
+    {
+      "epoch": 3.8871951219512195,
+      "grad_norm": 1.0020835399627686,
+      "learning_rate": 1.800616501737224e-06,
+      "loss": 0.1048,
+      "step": 427
+    },
+    {
+      "epoch": 3.8963414634146343,
+      "grad_norm": 0.8066243529319763,
+      "learning_rate": 1.7870170129584191e-06,
+      "loss": 0.1017,
+      "step": 428
+    },
+    {
+      "epoch": 3.9054878048780486,
+      "grad_norm": 0.6243605613708496,
+      "learning_rate": 1.7734404517597004e-06,
+      "loss": 0.1048,
+      "step": 429
+    },
+    {
+      "epoch": 3.9146341463414633,
+      "grad_norm": 0.7325178384780884,
+      "learning_rate": 1.7598872547260904e-06,
+      "loss": 0.1018,
+      "step": 430
+    },
+    {
+      "epoch": 3.923780487804878,
+      "grad_norm": 0.8298372030258179,
+      "learning_rate": 1.7463578576912859e-06,
+      "loss": 0.1061,
+      "step": 431
+    },
+    {
+      "epoch": 3.932926829268293,
+      "grad_norm": 0.3550826609134674,
+      "learning_rate": 1.7328526957236408e-06,
+      "loss": 0.105,
+      "step": 432
+    },
+    {
+      "epoch": 3.942073170731707,
+      "grad_norm": 0.6778271794319153,
+      "learning_rate": 1.7193722031121726e-06,
+      "loss": 0.1007,
+      "step": 433
+    },
+    {
+      "epoch": 3.951219512195122,
+      "grad_norm": 0.8127948641777039,
+      "learning_rate": 1.7059168133526043e-06,
+      "loss": 0.1009,
+      "step": 434
+    },
+    {
+      "epoch": 3.9603658536585367,
+      "grad_norm": 0.3050505518913269,
+      "learning_rate": 1.692486959133417e-06,
+      "loss": 0.1035,
+      "step": 435
+    },
+    {
+      "epoch": 3.9695121951219514,
+      "grad_norm": 0.6452750563621521,
+      "learning_rate": 1.6790830723219398e-06,
+      "loss": 0.1061,
+      "step": 436
+    },
+    {
+      "epoch": 3.9786585365853657,
+      "grad_norm": 0.72505784034729,
+      "learning_rate": 1.6657055839504601e-06,
+      "loss": 0.1041,
+      "step": 437
+    },
+    {
+      "epoch": 3.9878048780487805,
+      "grad_norm": 1.2222975492477417,
+      "learning_rate": 1.6523549242023657e-06,
+      "loss": 0.1025,
+      "step": 438
+    },
+    {
+      "epoch": 3.9969512195121952,
+      "grad_norm": 0.8575263619422913,
+      "learning_rate": 1.6390315223983068e-06,
+      "loss": 0.1004,
+      "step": 439
+    },
+    {
+      "epoch": 4.0,
+      "grad_norm": 0.8575263619422913,
+      "learning_rate": 1.6257358069823965e-06,
+      "loss": 0.0363,
+      "step": 440
+    },
+    {
+      "epoch": 4.009146341463414,
+      "grad_norm": 0.9300926327705383,
+      "learning_rate": 1.6124682055084245e-06,
+      "loss": 0.1052,
+      "step": 441
+    },
+    {
+      "epoch": 4.0182926829268295,
+      "grad_norm": 0.7189462780952454,
+      "learning_rate": 1.599229144626117e-06,
+      "loss": 0.1051,
+      "step": 442
+    },
+    {
+      "epoch": 4.027439024390244,
+      "grad_norm": 0.5437363386154175,
+      "learning_rate": 1.5860190500674115e-06,
+      "loss": 0.1024,
+      "step": 443
+    },
+    {
+      "epoch": 4.036585365853658,
+      "grad_norm": 0.39869755506515503,
+      "learning_rate": 1.5728383466327685e-06,
+      "loss": 0.1039,
+      "step": 444
+    },
+    {
+      "epoch": 4.045731707317073,
+      "grad_norm": 0.9101700186729431,
+      "learning_rate": 1.5596874581775113e-06,
+      "loss": 0.104,
+      "step": 445
+    },
+    {
+      "epoch": 4.054878048780488,
+      "grad_norm": 0.8250009417533875,
+      "learning_rate": 1.5465668075981905e-06,
+      "loss": 0.1022,
+      "step": 446
+    },
+    {
+      "epoch": 4.064024390243903,
+      "grad_norm": 0.31336626410484314,
+      "learning_rate": 1.5334768168189953e-06,
+      "loss": 0.1027,
+      "step": 447
+    },
+    {
+      "epoch": 4.073170731707317,
+      "grad_norm": 0.807352602481842,
+      "learning_rate": 1.520417906778176e-06,
+      "loss": 0.1015,
+      "step": 448
+    },
+    {
+      "epoch": 4.0823170731707314,
+      "grad_norm": 1.00467848777771,
+      "learning_rate": 1.5073904974145145e-06,
+      "loss": 0.1035,
+      "step": 449
+    },
+    {
+      "epoch": 4.091463414634147,
+      "grad_norm": 0.42968934774398804,
+      "learning_rate": 1.4943950076538135e-06,
+      "loss": 0.102,
+      "step": 450
+    },
+    {
+      "epoch": 4.100609756097561,
+      "grad_norm": 0.573142409324646,
+      "learning_rate": 1.4814318553954307e-06,
+      "loss": 0.1064,
+      "step": 451
+    },
+    {
+      "epoch": 4.109756097560975,
+      "grad_norm": 0.7983995079994202,
+      "learning_rate": 1.4685014574988394e-06,
+      "loss": 0.106,
+      "step": 452
+    },
+    {
+      "epoch": 4.1189024390243905,
+      "grad_norm": 0.7572171092033386,
+      "learning_rate": 1.455604229770221e-06,
+      "loss": 0.1033,
+      "step": 453
+    },
+    {
+      "epoch": 4.128048780487805,
+      "grad_norm": 1.041406273841858,
+      "learning_rate": 1.4427405869490924e-06,
+      "loss": 0.1084,
+      "step": 454
+    },
+    {
+      "epoch": 4.137195121951219,
+      "grad_norm": 0.9685583114624023,
+      "learning_rate": 1.4299109426949784e-06,
+      "loss": 0.1005,
+      "step": 455
+    },
+    {
+      "epoch": 4.146341463414634,
+      "grad_norm": 0.9551640748977661,
+      "learning_rate": 1.4171157095740976e-06,
+      "loss": 0.0999,
+      "step": 456
+    },
+    {
+      "epoch": 4.155487804878049,
+      "grad_norm": 0.8675352334976196,
+      "learning_rate": 1.404355299046103e-06,
+      "loss": 0.1078,
+      "step": 457
+    },
+    {
+      "epoch": 4.164634146341464,
+      "grad_norm": 0.4849691390991211,
+      "learning_rate": 1.391630121450847e-06,
+      "loss": 0.1015,
+      "step": 458
+    },
+    {
+      "epoch": 4.173780487804878,
+      "grad_norm": 0.4850751757621765,
+      "learning_rate": 1.3789405859951894e-06,
+      "loss": 0.1029,
+      "step": 459
+    },
+    {
+      "epoch": 4.182926829268292,
+      "grad_norm": 0.4103759825229645,
+      "learning_rate": 1.3662871007398347e-06,
+      "loss": 0.1007,
+      "step": 460
+    },
+    {
+      "epoch": 4.192073170731708,
+      "grad_norm": 0.4612462520599365,
+      "learning_rate": 1.3536700725862115e-06,
+      "loss": 0.1036,
+      "step": 461
+    },
+    {
+      "epoch": 4.201219512195122,
+      "grad_norm": 0.4885130524635315,
+      "learning_rate": 1.3410899072633915e-06,
+      "loss": 0.1046,
+      "step": 462
+    },
+    {
+      "epoch": 4.210365853658536,
+      "grad_norm": 0.3064654767513275,
+      "learning_rate": 1.3285470093150328e-06,
+      "loss": 0.0992,
+      "step": 463
+    },
+    {
+      "epoch": 4.219512195121951,
+      "grad_norm": 0.76682448387146,
+      "learning_rate": 1.3160417820863808e-06,
+      "loss": 0.1051,
+      "step": 464
+    },
+    {
+      "epoch": 4.228658536585366,
+      "grad_norm": 0.771308958530426,
+      "learning_rate": 1.303574627711292e-06,
+      "loss": 0.1028,
+      "step": 465
+    },
+    {
+      "epoch": 4.237804878048781,
+      "grad_norm": 0.327519029378891,
+      "learning_rate": 1.2911459470993084e-06,
+      "loss": 0.1025,
+      "step": 466
+    },
+    {
+      "epoch": 4.246951219512195,
+      "grad_norm": 0.5206422209739685,
+      "learning_rate": 1.2787561399227545e-06,
+      "loss": 0.103,
+      "step": 467
+    },
+    {
+      "epoch": 4.2560975609756095,
+      "grad_norm": 0.7346555590629578,
+      "learning_rate": 1.2664056046038941e-06,
+      "loss": 0.1042,
+      "step": 468
+    },
+    {
+      "epoch": 4.265243902439025,
+      "grad_norm": 0.9764044880867004,
+      "learning_rate": 1.2540947383021197e-06,
+      "loss": 0.0985,
+      "step": 469
+    },
+    {
+      "epoch": 4.274390243902439,
+      "grad_norm": 0.8075615167617798,
+      "learning_rate": 1.2418239369011712e-06,
+      "loss": 0.1038,
+      "step": 470
+    },
+    {
+      "epoch": 4.283536585365853,
+      "grad_norm": 0.7572922706604004,
+      "learning_rate": 1.2295935949964143e-06,
+      "loss": 0.1044,
+      "step": 471
+    },
+    {
+      "epoch": 4.2926829268292686,
+      "grad_norm": 0.6829261183738708,
+      "learning_rate": 1.217404105882147e-06,
+      "loss": 0.101,
+      "step": 472
+    },
+    {
+      "epoch": 4.301829268292683,
+      "grad_norm": 0.2822739779949188,
+      "learning_rate": 1.2052558615389535e-06,
+      "loss": 0.1043,
+      "step": 473
+    },
+    {
+      "epoch": 4.310975609756097,
+      "grad_norm": 0.3203172981739044,
+      "learning_rate": 1.193149252621099e-06,
+      "loss": 0.1021,
+      "step": 474
+    },
+    {
+      "epoch": 4.320121951219512,
+      "grad_norm": 0.8120080828666687,
+      "learning_rate": 1.1810846684439682e-06,
+      "loss": 0.1035,
+      "step": 475
+    },
+    {
+      "epoch": 4.329268292682927,
+      "grad_norm": 0.5470507740974426,
+      "learning_rate": 1.1690624969715441e-06,
+      "loss": 0.1013,
+      "step": 476
+    },
+    {
+      "epoch": 4.338414634146342,
+      "grad_norm": 0.7421087026596069,
+      "learning_rate": 1.1570831248039347e-06,
+      "loss": 0.1029,
+      "step": 477
+    },
+    {
+      "epoch": 4.347560975609756,
+      "grad_norm": 1.4894626140594482,
+      "learning_rate": 1.1451469371649382e-06,
+      "loss": 0.1007,
+      "step": 478
+    },
+    {
+      "epoch": 4.3567073170731705,
+      "grad_norm": 0.8726413249969482,
+      "learning_rate": 1.1332543178896578e-06,
+      "loss": 0.1035,
+      "step": 479
+    },
+    {
+      "epoch": 4.365853658536586,
+      "grad_norm": 0.3398156464099884,
+      "learning_rate": 1.121405649412156e-06,
+      "loss": 0.1049,
+      "step": 480
+    },
+    {
+      "epoch": 4.375,
+      "grad_norm": 1.0846507549285889,
+      "learning_rate": 1.10960131275316e-06,
+      "loss": 0.1038,
+      "step": 481
+    },
+    {
+      "epoch": 4.384146341463414,
+      "grad_norm": 0.7278075814247131,
+      "learning_rate": 1.0978416875078043e-06,
+      "loss": 0.1045,
+      "step": 482
+    },
+    {
+      "epoch": 4.3932926829268295,
+      "grad_norm": 0.6897367835044861,
+      "learning_rate": 1.0861271518334306e-06,
+      "loss": 0.1026,
+      "step": 483
+    },
+    {
+      "epoch": 4.402439024390244,
+      "grad_norm": 0.5504060387611389,
+      "learning_rate": 1.0744580824374218e-06,
+      "loss": 0.0984,
+      "step": 484
+    },
+    {
+      "epoch": 4.411585365853658,
+      "grad_norm": 0.3768717646598816,
+      "learning_rate": 1.062834854565087e-06,
+      "loss": 0.1028,
+      "step": 485
+    },
+    {
+      "epoch": 4.420731707317073,
+      "grad_norm": 0.6275299191474915,
+      "learning_rate": 1.0512578419876005e-06,
+      "loss": 0.1022,
+      "step": 486
+    },
+    {
+      "epoch": 4.429878048780488,
+      "grad_norm": 0.6650844216346741,
+      "learning_rate": 1.0397274169899802e-06,
+      "loss": 0.1024,
+      "step": 487
+    },
+    {
+      "epoch": 4.439024390243903,
+      "grad_norm": 0.5434347987174988,
+      "learning_rate": 1.0282439503591135e-06,
+      "loss": 0.1013,
+      "step": 488
+    },
+    {
+      "epoch": 4.448170731707317,
+      "grad_norm": 1.1153298616409302,
+      "learning_rate": 1.0168078113718327e-06,
+      "loss": 0.1027,
+      "step": 489
+    },
+    {
+      "epoch": 4.4573170731707314,
+      "grad_norm": 0.6244189143180847,
+      "learning_rate": 1.005419367783048e-06,
+      "loss": 0.1009,
+      "step": 490
+    },
+    {
+      "epoch": 4.466463414634147,
+      "grad_norm": 0.46386802196502686,
+      "learning_rate": 9.940789858139134e-07,
+      "loss": 0.0998,
+      "step": 491
+    },
+    {
+      "epoch": 4.475609756097561,
+      "grad_norm": 0.8419226408004761,
+      "learning_rate": 9.827870301400528e-07,
+      "loss": 0.1041,
+      "step": 492
+    },
+    {
+      "epoch": 4.484756097560975,
+      "grad_norm": 0.4894380271434784,
+      "learning_rate": 9.715438638798315e-07,
+      "loss": 0.1029,
+      "step": 493
+    },
+    {
+      "epoch": 4.4939024390243905,
+      "grad_norm": 1.153964638710022,
+      "learning_rate": 9.60349848582685e-07,
+      "loss": 0.1057,
+      "step": 494
+    },
+    {
+      "epoch": 4.503048780487805,
+      "grad_norm": 0.3604268729686737,
+      "learning_rate": 9.492053442174851e-07,
+      "loss": 0.102,
+      "step": 495
+    },
+    {
+      "epoch": 4.512195121951219,
+      "grad_norm": 0.9208924770355225,
+      "learning_rate": 9.381107091609689e-07,
+      "loss": 0.1025,
+      "step": 496
+    },
+    {
+      "epoch": 4.521341463414634,
+      "grad_norm": 0.47151800990104675,
+      "learning_rate": 9.270663001862129e-07,
+      "loss": 0.1031,
+      "step": 497
+    },
+    {
+      "epoch": 4.530487804878049,
+      "grad_norm": 0.3480950891971588,
+      "learning_rate": 9.160724724511608e-07,
+      "loss": 0.0968,
+      "step": 498
+    },
+    {
+      "epoch": 4.539634146341464,
+      "grad_norm": 0.3497055470943451,
+      "learning_rate": 9.051295794872008e-07,
+      "loss": 0.102,
+      "step": 499
+    },
+    {
+      "epoch": 4.548780487804878,
+      "grad_norm": 0.48209884762763977,
+      "learning_rate": 8.942379731877992e-07,
+      "loss": 0.1047,
+      "step": 500
+    },
+    {
+      "epoch": 4.557926829268292,
+      "grad_norm": 0.4844468832015991,
+      "learning_rate": 8.833980037971862e-07,
+      "loss": 0.1007,
+      "step": 501
+    },
+    {
+      "epoch": 4.567073170731708,
+      "grad_norm": 0.5743635296821594,
+      "learning_rate": 8.726100198990853e-07,
+      "loss": 0.1013,
+      "step": 502
+    },
+    {
+      "epoch": 4.576219512195122,
+      "grad_norm": 0.8410227298736572,
+      "learning_rate": 8.618743684055125e-07,
+      "loss": 0.1021,
+      "step": 503
+    },
+    {
+      "epoch": 4.585365853658536,
+      "grad_norm": 0.7356903553009033,
+      "learning_rate": 8.511913945456151e-07,
+      "loss": 0.0994,
+      "step": 504
+    },
+    {
+      "epoch": 4.594512195121951,
+      "grad_norm": 0.7952869534492493,
+      "learning_rate": 8.40561441854576e-07,
+      "loss": 0.1039,
+      "step": 505
+    },
+    {
+      "epoch": 4.603658536585366,
+      "grad_norm": 0.7460613250732422,
+      "learning_rate": 8.299848521625564e-07,
+      "loss": 0.0997,
+      "step": 506
+    },
+    {
+      "epoch": 4.612804878048781,
+      "grad_norm": 0.4789368510246277,
+      "learning_rate": 8.194619655837135e-07,
+      "loss": 0.0988,
+      "step": 507
+    },
+    {
+      "epoch": 4.621951219512195,
+      "grad_norm": 0.4485442638397217,
+      "learning_rate": 8.089931205052598e-07,
+      "loss": 0.1014,
+      "step": 508
+    },
+    {
+      "epoch": 4.6310975609756095,
+      "grad_norm": 0.36737513542175293,
+      "learning_rate": 7.985786535765794e-07,
+      "loss": 0.0996,
+      "step": 509
+    },
+    {
+      "epoch": 4.640243902439025,
+      "grad_norm": 0.6417019963264465,
+      "learning_rate": 7.882188996984047e-07,
+      "loss": 0.1026,
+      "step": 510
+    },
+    {
+      "epoch": 4.649390243902439,
+      "grad_norm": 0.6503584980964661,
+      "learning_rate": 7.779141920120462e-07,
+      "loss": 0.0981,
+      "step": 511
+    },
+    {
+      "epoch": 4.658536585365853,
+      "grad_norm": 0.3724409341812134,
+      "learning_rate": 7.676648618886797e-07,
+      "loss": 0.1001,
+      "step": 512
+    },
+    {
+      "epoch": 4.6676829268292686,
+      "grad_norm": 0.8001664876937866,
+      "learning_rate": 7.574712389186906e-07,
+      "loss": 0.1014,
+      "step": 513
+    },
+    {
+      "epoch": 4.676829268292683,
+      "grad_norm": 0.48333311080932617,
+      "learning_rate": 7.473336509010742e-07,
+      "loss": 0.098,
+      "step": 514
+    },
+    {
+      "epoch": 4.685975609756097,
+      "grad_norm": 1.0942858457565308,
+      "learning_rate": 7.372524238328955e-07,
+      "loss": 0.1,
+      "step": 515
+    },
+    {
+      "epoch": 4.695121951219512,
+      "grad_norm": 0.48746439814567566,
+      "learning_rate": 7.272278818988054e-07,
+      "loss": 0.1015,
+      "step": 516
+    },
+    {
+      "epoch": 4.704268292682927,
+      "grad_norm": 0.539008617401123,
+      "learning_rate": 7.172603474606169e-07,
+      "loss": 0.1001,
+      "step": 517
+    },
+    {
+      "epoch": 4.713414634146341,
+      "grad_norm": 0.36583229899406433,
+      "learning_rate": 7.073501410469371e-07,
+      "loss": 0.1006,
+      "step": 518
+    },
+    {
+      "epoch": 4.722560975609756,
+      "grad_norm": 0.6509392261505127,
+      "learning_rate": 6.974975813428622e-07,
+      "loss": 0.1006,
+      "step": 519
+    },
+    {
+      "epoch": 4.7317073170731705,
+      "grad_norm": 0.5271562337875366,
+      "learning_rate": 6.877029851797265e-07,
+      "loss": 0.1021,
+      "step": 520
+    },
+    {
+      "epoch": 4.740853658536586,
+      "grad_norm": 0.530583918094635,
+      "learning_rate": 6.779666675249166e-07,
+      "loss": 0.1023,
+      "step": 521
+    },
+    {
+      "epoch": 4.75,
+      "grad_norm": 0.6522917747497559,
+      "learning_rate": 6.682889414717417e-07,
+      "loss": 0.0987,
+      "step": 522
+    },
+    {
+      "epoch": 4.759146341463414,
+      "grad_norm": 0.3822062611579895,
+      "learning_rate": 6.586701182293654e-07,
+      "loss": 0.1004,
+      "step": 523
+    },
+    {
+      "epoch": 4.7682926829268295,
+      "grad_norm": 0.38580814003944397,
+      "learning_rate": 6.491105071127985e-07,
+      "loss": 0.0971,
+      "step": 524
+    },
+    {
+      "epoch": 4.777439024390244,
+      "grad_norm": 0.43107372522354126,
+      "learning_rate": 6.396104155329508e-07,
+      "loss": 0.098,
+      "step": 525
+    },
+    {
+      "epoch": 4.786585365853659,
+      "grad_norm": 0.5299065113067627,
+      "learning_rate": 6.301701489867496e-07,
+      "loss": 0.0997,
+      "step": 526
+    },
+    {
+      "epoch": 4.795731707317073,
+      "grad_norm": 0.7257470488548279,
+      "learning_rate": 6.207900110473112e-07,
+      "loss": 0.0962,
+      "step": 527
+    },
+    {
+      "epoch": 4.804878048780488,
+      "grad_norm": 0.7482612133026123,
+      "learning_rate": 6.114703033541783e-07,
+      "loss": 0.1027,
+      "step": 528
+    },
+    {
+      "epoch": 4.814024390243903,
+      "grad_norm": 0.4249897301197052,
+      "learning_rate": 6.022113256036268e-07,
+      "loss": 0.102,
+      "step": 529
+    },
+    {
+      "epoch": 4.823170731707317,
+      "grad_norm": 0.34573644399642944,
+      "learning_rate": 5.930133755390216e-07,
+      "loss": 0.1018,
+      "step": 530
+    },
+    {
+      "epoch": 4.8323170731707314,
+      "grad_norm": 0.6430623531341553,
+      "learning_rate": 5.838767489412453e-07,
+      "loss": 0.0993,
+      "step": 531
+    },
+    {
+      "epoch": 4.841463414634147,
+      "grad_norm": 0.3944159150123596,
+      "learning_rate": 5.748017396191827e-07,
+      "loss": 0.0999,
+      "step": 532
+    },
+    {
+      "epoch": 4.850609756097561,
+      "grad_norm": 0.5905594229698181,
+      "learning_rate": 5.657886394002802e-07,
+      "loss": 0.1013,
+      "step": 533
+    },
+    {
+      "epoch": 4.859756097560975,
+      "grad_norm": 0.6324522495269775,
+      "learning_rate": 5.568377381211548e-07,
+      "loss": 0.1026,
+      "step": 534
+    },
+    {
+      "epoch": 4.8689024390243905,
+      "grad_norm": 0.6493394374847412,
+      "learning_rate": 5.479493236182759e-07,
+      "loss": 0.1015,
+      "step": 535
+    },
+    {
+      "epoch": 4.878048780487805,
+      "grad_norm": 0.7953840494155884,
+      "learning_rate": 5.391236817187096e-07,
+      "loss": 0.1026,
+      "step": 536
+    },
+    {
+      "epoch": 4.887195121951219,
+      "grad_norm": 0.46242570877075195,
+      "learning_rate": 5.303610962309266e-07,
+      "loss": 0.0993,
+      "step": 537
+    },
+    {
+      "epoch": 4.896341463414634,
+      "grad_norm": 0.6665740609169006,
+      "learning_rate": 5.216618489356773e-07,
+      "loss": 0.1015,
+      "step": 538
+    },
+    {
+      "epoch": 4.905487804878049,
+      "grad_norm": 0.5248611569404602,
+      "learning_rate": 5.130262195769273e-07,
+      "loss": 0.1029,
+      "step": 539
+    },
+    {
+      "epoch": 4.914634146341464,
+      "grad_norm": 0.5713643431663513,
+      "learning_rate": 5.044544858528668e-07,
+      "loss": 0.1021,
+      "step": 540
+    },
+    {
+      "epoch": 4.923780487804878,
+      "grad_norm": 0.7313759326934814,
+      "learning_rate": 4.959469234069735e-07,
+      "loss": 0.0989,
+      "step": 541
+    },
+    {
+      "epoch": 4.932926829268292,
+      "grad_norm": 0.7184482216835022,
+      "learning_rate": 4.87503805819155e-07,
+      "loss": 0.0981,
+      "step": 542
+    },
+    {
+      "epoch": 4.942073170731708,
+      "grad_norm": 0.4711061716079712,
+      "learning_rate": 4.791254045969477e-07,
+      "loss": 0.1,
+      "step": 543
+    },
+    {
+      "epoch": 4.951219512195122,
+      "grad_norm": 0.4654499888420105,
+      "learning_rate": 4.7081198916678924e-07,
+      "loss": 0.0992,
+      "step": 544
+    },
+    {
+      "epoch": 4.960365853658536,
+      "grad_norm": 1.0146021842956543,
+      "learning_rate": 4.6256382686534844e-07,
+      "loss": 0.0993,
+      "step": 545
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 654,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 6,
+  "save_steps": 109,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.6460245175877239e+19,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}
diff --git a/checkpoint-545/training_args.bin b/checkpoint-545/training_args.bin
new file mode 100644
index 0000000000000000000000000000000000000000..24b9be216f1966bb6d081a79c30a21dced39cf99
--- /dev/null
+++ b/checkpoint-545/training_args.bin
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:14990b8399bb8d39b746405b094f7eea1041d93488de8433b2d415de16ff957a
+size 7928
diff --git a/checkpoint-545/zero_to_fp32.py b/checkpoint-545/zero_to_fp32.py
new file mode 100644
index 0000000000000000000000000000000000000000..24cc342e78d1a006c782b3a4cd68d9ce786d8fd8
--- /dev/null
+++ b/checkpoint-545/zero_to_fp32.py
@@ -0,0 +1,604 @@
+#!/usr/bin/env python
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
+# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
+# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
+# application.
+#
+# example: python zero_to_fp32.py . pytorch_model.bin
+
+import argparse
+import torch
+import glob
+import math
+import os
+import re
+from collections import OrderedDict
+from dataclasses import dataclass
+
+# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
+# DeepSpeed data structures it has to be available in the current python environment.
+from deepspeed.utils import logger
+from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
+                                            FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
+                                            FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS)
+
+
+@dataclass
+class zero_model_state:
+    buffers: dict()
+    param_shapes: dict()
+    shared_params: list
+    ds_version: int
+    frozen_param_shapes: dict()
+    frozen_param_fragments: dict()
+
+
+debug = 0
+
+# load to cpu
+device = torch.device('cpu')
+
+
+def atoi(text):
+    return int(text) if text.isdigit() else text
+
+
+def natural_keys(text):
+    '''
+    alist.sort(key=natural_keys) sorts in human order
+    http://nedbatchelder.com/blog/200712/human_sorting.html
+    (See Toothy's implementation in the comments)
+    '''
+    return [atoi(c) for c in re.split(r'(\d+)', text)]
+
+
+def get_model_state_file(checkpoint_dir, zero_stage):
+    if not os.path.isdir(checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
+
+    # there should be only one file
+    if zero_stage <= 2:
+        file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
+    elif zero_stage == 3:
+        file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
+
+    if not os.path.exists(file):
+        raise FileNotFoundError(f"can't find model states file at '{file}'")
+
+    return file
+
+
+def get_checkpoint_files(checkpoint_dir, glob_pattern):
+    # XXX: need to test that this simple glob rule works for multi-node setup too
+    ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
+
+    if len(ckpt_files) == 0:
+        raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
+
+    return ckpt_files
+
+
+def get_optim_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
+
+
+def get_model_state_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
+
+
+def parse_model_states(files):
+    zero_model_states = []
+    for file in files:
+        state_dict = torch.load(file, map_location=device)
+
+        if BUFFER_NAMES not in state_dict:
+            raise ValueError(f"{file} is not a model state checkpoint")
+        buffer_names = state_dict[BUFFER_NAMES]
+        if debug:
+            print("Found buffers:", buffer_names)
+
+        # recover just the buffers while restoring them to fp32 if they were saved in fp16
+        buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
+        param_shapes = state_dict[PARAM_SHAPES]
+
+        # collect parameters that are included in param_shapes
+        param_names = []
+        for s in param_shapes:
+            for name in s.keys():
+                param_names.append(name)
+
+        # update with frozen parameters
+        frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
+        if frozen_param_shapes is not None:
+            if debug:
+                print(f"Found frozen_param_shapes: {frozen_param_shapes}")
+            param_names += list(frozen_param_shapes.keys())
+
+        # handle shared params
+        shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
+
+        ds_version = state_dict.get(DS_VERSION, None)
+
+        frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
+
+        z_model_state = zero_model_state(buffers=buffers,
+                                         param_shapes=param_shapes,
+                                         shared_params=shared_params,
+                                         ds_version=ds_version,
+                                         frozen_param_shapes=frozen_param_shapes,
+                                         frozen_param_fragments=frozen_param_fragments)
+        zero_model_states.append(z_model_state)
+
+    return zero_model_states
+
+
+def parse_optim_states(files, ds_checkpoint_dir):
+
+    total_files = len(files)
+    state_dicts = []
+    for f in files:
+        state_dict = torch.load(f, map_location=device)
+        # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+        # and also handle the case where it was already removed by another helper script
+        state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+        state_dicts.append(state_dict)
+
+    if not ZERO_STAGE in state_dicts[0][OPTIMIZER_STATE_DICT]:
+        raise ValueError(f"{files[0]} is not a zero checkpoint")
+    zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
+    world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
+
+    # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
+    # parameters can be different from data parallelism for non-expert parameters. So we can just
+    # use the max of the partition_count to get the dp world_size.
+
+    if type(world_size) is list:
+        world_size = max(world_size)
+
+    if world_size != total_files:
+        raise ValueError(
+            f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
+            "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
+        )
+
+    # the groups are named differently in each stage
+    if zero_stage <= 2:
+        fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
+    elif zero_stage == 3:
+        fp32_groups_key = FP32_FLAT_GROUPS
+    else:
+        raise ValueError(f"unknown zero stage {zero_stage}")
+
+    if zero_stage <= 2:
+        fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
+    elif zero_stage == 3:
+        # if there is more than one param group, there will be multiple flattened tensors - one
+        # flattened tensor per group - for simplicity merge them into a single tensor
+        #
+        # XXX: could make the script more memory efficient for when there are multiple groups - it
+        # will require matching the sub-lists of param_shapes for each param group flattened tensor
+
+        fp32_flat_groups = [
+            torch.cat(state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key], 0) for i in range(len(state_dicts))
+        ]
+
+    return zero_stage, world_size, fp32_flat_groups
+
+
+def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
+    """
+    Returns fp32 state_dict reconstructed from ds checkpoint
+
+    Args:
+        - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
+
+    """
+    print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
+
+    optim_files = get_optim_files(ds_checkpoint_dir)
+    zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
+    print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
+
+    model_files = get_model_state_files(ds_checkpoint_dir)
+
+    zero_model_states = parse_model_states(model_files)
+    print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
+
+    if zero_stage <= 2:
+        return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+    elif zero_stage == 3:
+        return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+
+
+def _zero2_merge_frozen_params(state_dict, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+    frozen_param_fragments = zero_model_states[0].frozen_param_fragments
+
+    if debug:
+        num_elem = sum(s.numel() for s in frozen_param_shapes.values())
+        print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        state_dict[name] = frozen_param_fragments[name]
+
+        if debug:
+            print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _has_callable(obj, fn):
+    attr = getattr(obj, fn, None)
+    return callable(attr)
+
+
+def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+
+    # Reconstruction protocol:
+    #
+    # XXX: document this
+
+    if debug:
+        for i in range(world_size):
+            for j in range(len(fp32_flat_groups[0])):
+                print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
+
+    # XXX: memory usage doubles here (zero2)
+    num_param_groups = len(fp32_flat_groups[0])
+    merged_single_partition_of_fp32_groups = []
+    for i in range(num_param_groups):
+        merged_partitions = [sd[i] for sd in fp32_flat_groups]
+        full_single_fp32_vector = torch.cat(merged_partitions, 0)
+        merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
+    avail_numel = sum(
+        [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
+
+    if debug:
+        wanted_params = sum([len(shapes) for shapes in param_shapes])
+        wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
+        # not asserting if there is a mismatch due to possible padding
+        print(f"Have {avail_numel} numels to process.")
+        print(f"Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    total_numel = 0
+    total_params = 0
+    for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
+        offset = 0
+        avail_numel = full_single_fp32_vector.numel()
+        for name, shape in shapes.items():
+
+            unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
+            total_numel += unpartitioned_numel
+            total_params += 1
+
+            if debug:
+                print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+            state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
+            offset += unpartitioned_numel
+
+        # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
+        # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
+        # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
+        # live optimizer object, so we are checking that the numbers are within the right range
+        align_to = 2 * world_size
+
+        def zero2_align(x):
+            return align_to * math.ceil(x / align_to)
+
+        if debug:
+            print(f"original offset={offset}, avail_numel={avail_numel}")
+
+        offset = zero2_align(offset)
+        avail_numel = zero2_align(avail_numel)
+
+        if debug:
+            print(f"aligned  offset={offset}, avail_numel={avail_numel}")
+
+        # Sanity check
+        if offset != avail_numel:
+            raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero2_merge_frozen_params(state_dict, zero_model_states)
+
+    _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def zero3_partitioned_param_info(unpartitioned_numel, world_size):
+    remainder = unpartitioned_numel % world_size
+    padding_numel = (world_size - remainder) if remainder else 0
+    partitioned_numel = math.ceil(unpartitioned_numel / world_size)
+    return partitioned_numel, padding_numel
+
+
+def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    if debug:
+        for i in range(world_size):
+            num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
+            print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in zero_model_states[0].frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
+        state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
+
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+    avail_numel = fp32_flat_groups[0].numel() * world_size
+    # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
+    # param, re-consolidating each param, while dealing with padding if any
+
+    # merge list of dicts, preserving order
+    param_shapes = {k: v for d in param_shapes for k, v in d.items()}
+
+    if debug:
+        for i in range(world_size):
+            print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
+
+        wanted_params = len(param_shapes)
+        wanted_numel = sum(shape.numel() for shape in param_shapes.values())
+        # not asserting if there is a mismatch due to possible padding
+        avail_numel = fp32_flat_groups[0].numel() * world_size
+        print(f"Trainable params: Have {avail_numel} numels to process.")
+        print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    offset = 0
+    total_numel = 0
+    total_params = 0
+    for name, shape in param_shapes.items():
+
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+        total_params += 1
+
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+        # XXX: memory usage doubles here
+        state_dict[name] = torch.cat(
+            tuple(fp32_flat_groups[i].narrow(0, offset, partitioned_numel) for i in range(world_size)),
+            0).narrow(0, 0, unpartitioned_numel).view(shape)
+        offset += partitioned_numel
+
+    offset *= world_size
+
+    # Sanity check
+    if offset != avail_numel:
+        raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
+
+    _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag=None, exclude_frozen_parameters=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
+    ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
+    via a model hub.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+
+    Returns:
+        - pytorch ``state_dict``
+
+    Note: this approach may not work if your application doesn't have sufficient free CPU memory and
+    you may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
+    the checkpoint.
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+        # do the training and checkpoint saving
+        state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
+        model = model.cpu() # move to cpu
+        model.load_state_dict(state_dict)
+        # submit to model hub or save the model to share with others
+
+    In this example the ``model`` will no longer be usable in the deepspeed context of the same
+    application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
+
+    """
+    if tag is None:
+        latest_path = os.path.join(checkpoint_dir, 'latest')
+        if os.path.isfile(latest_path):
+            with open(latest_path, 'r') as fd:
+                tag = fd.read().strip()
+        else:
+            raise ValueError(f"Unable to find 'latest' file at {latest_path}")
+
+    ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
+
+    if not os.path.isdir(ds_checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
+
+    return _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
+
+
+def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, output_file, tag=None, exclude_frozen_parameters=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
+    loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``output_file``: path to the pytorch fp32 state_dict output file (e.g. path/pytorch_model.bin)
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+    """
+
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag, exclude_frozen_parameters)
+    print(f"Saving fp32 state dict to {output_file}")
+    torch.save(state_dict, output_file)
+
+
+def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
+    """
+    1. Put the provided model to cpu
+    2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
+    3. Load it into the provided model
+
+    Args:
+        - ``model``: the model object to update
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+
+    Returns:
+        - ``model`: modified model
+
+    Make sure you have plenty of CPU memory available before you call this function. If you don't
+    have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
+    conveniently placed for you in the checkpoint folder.
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
+        model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
+        # submit to model hub or save the model to share with others
+
+    Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
+    of the same application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    """
+    logger.info(f"Extracting fp32 weights")
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+
+    logger.info(f"Overwriting model with fp32 weights")
+    model = model.cpu()
+    model.load_state_dict(state_dict, strict=False)
+
+    return model
+
+
+if __name__ == "__main__":
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("checkpoint_dir",
+                        type=str,
+                        help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+    parser.add_argument(
+        "output_file",
+        type=str,
+        help="path to the pytorch fp32 state_dict output file (e.g. path/checkpoint-12/pytorch_model.bin)")
+    parser.add_argument("-t",
+                        "--tag",
+                        type=str,
+                        default=None,
+                        help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
+    parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+    parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
+    args = parser.parse_args()
+
+    debug = args.debug
+
+    convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
+                                               args.output_file,
+                                               tag=args.tag,
+                                               exclude_frozen_parameters=args.exclude_frozen_parameters)
diff --git a/config.json b/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..b7324bbcd3035a34c2ac96f0e2a46dd94a5db25c
--- /dev/null
+++ b/config.json
@@ -0,0 +1,1497 @@
+{
+  "_attn_implementation_autoset": true,
+  "_name_or_path": "nvidia/Llama-3_3-Nemotron-Super-49B-v1",
+  "architectures": [
+    "DeciLMForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "auto_map": {
+    "AutoConfig": "nvidia/Llama-3_3-Nemotron-Super-49B-v1--configuration_decilm.DeciLMConfig",
+    "AutoModelForCausalLM": "nvidia/Llama-3_3-Nemotron-Super-49B-v1--modeling_decilm.DeciLMForCausalLM"
+  },
+  "block_configs": [
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "num_sink_tokens": null,
+        "replace_with_linear": false,
+        "sparsify": null,
+        "unshifted_sink": false,
+        "use_prefill_window_in_sink_attention": false,
+        "window_length": null
+      },
+      "ffn": {
+        "ffn_mult": 2.625,
+        "no_op": false,
+        "replace_with_linear": false,
+        "sparsify": null
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "num_sink_tokens": null,
+        "replace_with_linear": false,
+        "sparsify": null,
+        "unshifted_sink": false,
+        "use_prefill_window_in_sink_attention": false,
+        "window_length": null
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false,
+        "sparsify": null
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "num_sink_tokens": null,
+        "replace_with_linear": false,
+        "sparsify": null,
+        "unshifted_sink": false,
+        "use_prefill_window_in_sink_attention": false,
+        "window_length": null
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false,
+        "sparsify": null
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "num_sink_tokens": null,
+        "replace_with_linear": false,
+        "sparsify": null,
+        "unshifted_sink": false,
+        "use_prefill_window_in_sink_attention": false,
+        "window_length": null
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false,
+        "sparsify": null
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "num_sink_tokens": null,
+        "replace_with_linear": false,
+        "sparsify": null,
+        "unshifted_sink": false,
+        "use_prefill_window_in_sink_attention": false,
+        "window_length": null
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false,
+        "sparsify": null
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "num_sink_tokens": null,
+        "replace_with_linear": false,
+        "sparsify": null,
+        "unshifted_sink": false,
+        "use_prefill_window_in_sink_attention": false,
+        "window_length": null
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false,
+        "sparsify": null
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": null,
+        "no_op": true,
+        "num_sink_tokens": null,
+        "replace_with_linear": false,
+        "sparsify": null,
+        "unshifted_sink": false,
+        "use_prefill_window_in_sink_attention": false,
+        "window_length": null
+      },
+      "ffn": {
+        "ffn_mult": 2.625,
+        "no_op": false,
+        "replace_with_linear": false,
+        "sparsify": null
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": null,
+        "no_op": true,
+        "num_sink_tokens": null,
+        "replace_with_linear": false,
+        "sparsify": null,
+        "unshifted_sink": false,
+        "use_prefill_window_in_sink_attention": false,
+        "window_length": null
+      },
+      "ffn": {
+        "ffn_mult": 2.625,
+        "no_op": false,
+        "replace_with_linear": false,
+        "sparsify": null
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "num_sink_tokens": null,
+        "replace_with_linear": false,
+        "sparsify": null,
+        "unshifted_sink": false,
+        "use_prefill_window_in_sink_attention": false,
+        "window_length": null
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false,
+        "sparsify": null
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "num_sink_tokens": null,
+        "replace_with_linear": false,
+        "sparsify": null,
+        "unshifted_sink": false,
+        "use_prefill_window_in_sink_attention": false,
+        "window_length": null
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false,
+        "sparsify": null
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "num_sink_tokens": null,
+        "replace_with_linear": false,
+        "sparsify": null,
+        "unshifted_sink": false,
+        "use_prefill_window_in_sink_attention": false,
+        "window_length": null
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false,
+        "sparsify": null
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": null,
+        "no_op": true,
+        "num_sink_tokens": null,
+        "replace_with_linear": false,
+        "sparsify": null,
+        "unshifted_sink": false,
+        "use_prefill_window_in_sink_attention": false,
+        "window_length": null
+      },
+      "ffn": {
+        "ffn_mult": 3.28125,
+        "no_op": false,
+        "replace_with_linear": false,
+        "sparsify": null
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "num_sink_tokens": null,
+        "replace_with_linear": false,
+        "sparsify": null,
+        "unshifted_sink": false,
+        "use_prefill_window_in_sink_attention": false,
+        "window_length": null
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false,
+        "sparsify": null
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "num_sink_tokens": null,
+        "replace_with_linear": false,
+        "sparsify": null,
+        "unshifted_sink": false,
+        "use_prefill_window_in_sink_attention": false,
+        "window_length": null
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false,
+        "sparsify": null
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "num_sink_tokens": null,
+        "replace_with_linear": false,
+        "sparsify": null,
+        "unshifted_sink": false,
+        "use_prefill_window_in_sink_attention": false,
+        "window_length": null
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false,
+        "sparsify": null
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "num_sink_tokens": null,
+        "replace_with_linear": false,
+        "sparsify": null,
+        "unshifted_sink": false,
+        "use_prefill_window_in_sink_attention": false,
+        "window_length": null
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false,
+        "sparsify": null
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "num_sink_tokens": null,
+        "replace_with_linear": false,
+        "sparsify": null,
+        "unshifted_sink": false,
+        "use_prefill_window_in_sink_attention": false,
+        "window_length": null
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false,
+        "sparsify": null
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "num_sink_tokens": null,
+        "replace_with_linear": false,
+        "sparsify": null,
+        "unshifted_sink": false,
+        "use_prefill_window_in_sink_attention": false,
+        "window_length": null
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false,
+        "sparsify": null
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "num_sink_tokens": null,
+        "replace_with_linear": false,
+        "sparsify": null,
+        "unshifted_sink": false,
+        "use_prefill_window_in_sink_attention": false,
+        "window_length": null
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false,
+        "sparsify": null
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "num_sink_tokens": null,
+        "replace_with_linear": false,
+        "sparsify": null,
+        "unshifted_sink": false,
+        "use_prefill_window_in_sink_attention": false,
+        "window_length": null
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false,
+        "sparsify": null
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "num_sink_tokens": null,
+        "replace_with_linear": false,
+        "sparsify": null,
+        "unshifted_sink": false,
+        "use_prefill_window_in_sink_attention": false,
+        "window_length": null
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false,
+        "sparsify": null
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "num_sink_tokens": null,
+        "replace_with_linear": false,
+        "sparsify": null,
+        "unshifted_sink": false,
+        "use_prefill_window_in_sink_attention": false,
+        "window_length": null
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false,
+        "sparsify": null
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "num_sink_tokens": null,
+        "replace_with_linear": false,
+        "sparsify": null,
+        "unshifted_sink": false,
+        "use_prefill_window_in_sink_attention": false,
+        "window_length": null
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false,
+        "sparsify": null
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "num_sink_tokens": null,
+        "replace_with_linear": false,
+        "sparsify": null,
+        "unshifted_sink": false,
+        "use_prefill_window_in_sink_attention": false,
+        "window_length": null
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false,
+        "sparsify": null
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "num_sink_tokens": null,
+        "replace_with_linear": false,
+        "sparsify": null,
+        "unshifted_sink": false,
+        "use_prefill_window_in_sink_attention": false,
+        "window_length": null
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false,
+        "sparsify": null
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "num_sink_tokens": null,
+        "replace_with_linear": false,
+        "sparsify": null,
+        "unshifted_sink": false,
+        "use_prefill_window_in_sink_attention": false,
+        "window_length": null
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false,
+        "sparsify": null
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "num_sink_tokens": null,
+        "replace_with_linear": false,
+        "sparsify": null,
+        "unshifted_sink": false,
+        "use_prefill_window_in_sink_attention": false,
+        "window_length": null
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false,
+        "sparsify": null
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "num_sink_tokens": null,
+        "replace_with_linear": false,
+        "sparsify": null,
+        "unshifted_sink": false,
+        "use_prefill_window_in_sink_attention": false,
+        "window_length": null
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false,
+        "sparsify": null
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "num_sink_tokens": null,
+        "replace_with_linear": false,
+        "sparsify": null,
+        "unshifted_sink": false,
+        "use_prefill_window_in_sink_attention": false,
+        "window_length": null
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false,
+        "sparsify": null
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "num_sink_tokens": null,
+        "replace_with_linear": false,
+        "sparsify": null,
+        "unshifted_sink": false,
+        "use_prefill_window_in_sink_attention": false,
+        "window_length": null
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false,
+        "sparsify": null
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "num_sink_tokens": null,
+        "replace_with_linear": false,
+        "sparsify": null,
+        "unshifted_sink": false,
+        "use_prefill_window_in_sink_attention": false,
+        "window_length": null
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false,
+        "sparsify": null
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "num_sink_tokens": null,
+        "replace_with_linear": false,
+        "sparsify": null,
+        "unshifted_sink": false,
+        "use_prefill_window_in_sink_attention": false,
+        "window_length": null
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false,
+        "sparsify": null
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "num_sink_tokens": null,
+        "replace_with_linear": false,
+        "sparsify": null,
+        "unshifted_sink": false,
+        "use_prefill_window_in_sink_attention": false,
+        "window_length": null
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false,
+        "sparsify": null
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "num_sink_tokens": null,
+        "replace_with_linear": false,
+        "sparsify": null,
+        "unshifted_sink": false,
+        "use_prefill_window_in_sink_attention": false,
+        "window_length": null
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false,
+        "sparsify": null
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "num_sink_tokens": null,
+        "replace_with_linear": false,
+        "sparsify": null,
+        "unshifted_sink": false,
+        "use_prefill_window_in_sink_attention": false,
+        "window_length": null
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false,
+        "sparsify": null
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "num_sink_tokens": null,
+        "replace_with_linear": false,
+        "sparsify": null,
+        "unshifted_sink": false,
+        "use_prefill_window_in_sink_attention": false,
+        "window_length": null
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false,
+        "sparsify": null
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "num_sink_tokens": null,
+        "replace_with_linear": false,
+        "sparsify": null,
+        "unshifted_sink": false,
+        "use_prefill_window_in_sink_attention": false,
+        "window_length": null
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false,
+        "sparsify": null
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "num_sink_tokens": null,
+        "replace_with_linear": false,
+        "sparsify": null,
+        "unshifted_sink": false,
+        "use_prefill_window_in_sink_attention": false,
+        "window_length": null
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false,
+        "sparsify": null
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "num_sink_tokens": null,
+        "replace_with_linear": false,
+        "sparsify": null,
+        "unshifted_sink": false,
+        "use_prefill_window_in_sink_attention": false,
+        "window_length": null
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false,
+        "sparsify": null
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "num_sink_tokens": null,
+        "replace_with_linear": false,
+        "sparsify": null,
+        "unshifted_sink": false,
+        "use_prefill_window_in_sink_attention": false,
+        "window_length": null
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false,
+        "sparsify": null
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "num_sink_tokens": null,
+        "replace_with_linear": false,
+        "sparsify": null,
+        "unshifted_sink": false,
+        "use_prefill_window_in_sink_attention": false,
+        "window_length": null
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false,
+        "sparsify": null
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "num_sink_tokens": null,
+        "replace_with_linear": false,
+        "sparsify": null,
+        "unshifted_sink": false,
+        "use_prefill_window_in_sink_attention": false,
+        "window_length": null
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false,
+        "sparsify": null
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": null,
+        "no_op": true,
+        "num_sink_tokens": null,
+        "replace_with_linear": false,
+        "sparsify": null,
+        "unshifted_sink": false,
+        "use_prefill_window_in_sink_attention": false,
+        "window_length": null
+      },
+      "ffn": {
+        "ffn_mult": 1.3125,
+        "no_op": false,
+        "replace_with_linear": false,
+        "sparsify": null
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": null,
+        "no_op": true,
+        "num_sink_tokens": null,
+        "replace_with_linear": false,
+        "sparsify": null,
+        "unshifted_sink": false,
+        "use_prefill_window_in_sink_attention": false,
+        "window_length": null
+      },
+      "ffn": {
+        "ffn_mult": 2.625,
+        "no_op": false,
+        "replace_with_linear": false,
+        "sparsify": null
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": null,
+        "no_op": true,
+        "num_sink_tokens": null,
+        "replace_with_linear": false,
+        "sparsify": null,
+        "unshifted_sink": false,
+        "use_prefill_window_in_sink_attention": false,
+        "window_length": null
+      },
+      "ffn": {
+        "ffn_mult": 2.625,
+        "no_op": false,
+        "replace_with_linear": false,
+        "sparsify": null
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": null,
+        "no_op": true,
+        "num_sink_tokens": null,
+        "replace_with_linear": false,
+        "sparsify": null,
+        "unshifted_sink": false,
+        "use_prefill_window_in_sink_attention": false,
+        "window_length": null
+      },
+      "ffn": {
+        "ffn_mult": 1.3125,
+        "no_op": false,
+        "replace_with_linear": false,
+        "sparsify": null
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": null,
+        "no_op": true,
+        "num_sink_tokens": null,
+        "replace_with_linear": false,
+        "sparsify": null,
+        "unshifted_sink": false,
+        "use_prefill_window_in_sink_attention": false,
+        "window_length": null
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false,
+        "sparsify": null
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": null,
+        "no_op": true,
+        "num_sink_tokens": null,
+        "replace_with_linear": false,
+        "sparsify": null,
+        "unshifted_sink": false,
+        "use_prefill_window_in_sink_attention": false,
+        "window_length": null
+      },
+      "ffn": {
+        "ffn_mult": 1.3125,
+        "no_op": false,
+        "replace_with_linear": false,
+        "sparsify": null
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": null,
+        "no_op": true,
+        "num_sink_tokens": null,
+        "replace_with_linear": false,
+        "sparsify": null,
+        "unshifted_sink": false,
+        "use_prefill_window_in_sink_attention": false,
+        "window_length": null
+      },
+      "ffn": {
+        "ffn_mult": 2.625,
+        "no_op": false,
+        "replace_with_linear": false,
+        "sparsify": null
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": null,
+        "no_op": true,
+        "num_sink_tokens": null,
+        "replace_with_linear": false,
+        "sparsify": null,
+        "unshifted_sink": false,
+        "use_prefill_window_in_sink_attention": false,
+        "window_length": null
+      },
+      "ffn": {
+        "ffn_mult": 1.3125,
+        "no_op": false,
+        "replace_with_linear": false,
+        "sparsify": null
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": null,
+        "no_op": true,
+        "num_sink_tokens": null,
+        "replace_with_linear": false,
+        "sparsify": null,
+        "unshifted_sink": false,
+        "use_prefill_window_in_sink_attention": false,
+        "window_length": null
+      },
+      "ffn": {
+        "ffn_mult": 1.3125,
+        "no_op": false,
+        "replace_with_linear": false,
+        "sparsify": null
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": null,
+        "no_op": true,
+        "num_sink_tokens": null,
+        "replace_with_linear": false,
+        "sparsify": null,
+        "unshifted_sink": false,
+        "use_prefill_window_in_sink_attention": false,
+        "window_length": null
+      },
+      "ffn": {
+        "ffn_mult": 1.3125,
+        "no_op": false,
+        "replace_with_linear": false,
+        "sparsify": null
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "num_sink_tokens": null,
+        "replace_with_linear": false,
+        "sparsify": null,
+        "unshifted_sink": false,
+        "use_prefill_window_in_sink_attention": false,
+        "window_length": null
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false,
+        "sparsify": null
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": null,
+        "no_op": true,
+        "num_sink_tokens": null,
+        "replace_with_linear": false,
+        "sparsify": null,
+        "unshifted_sink": false,
+        "use_prefill_window_in_sink_attention": false,
+        "window_length": null
+      },
+      "ffn": {
+        "ffn_mult": 1.3125,
+        "no_op": false,
+        "replace_with_linear": false,
+        "sparsify": null
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": null,
+        "no_op": true,
+        "num_sink_tokens": null,
+        "replace_with_linear": false,
+        "sparsify": null,
+        "unshifted_sink": false,
+        "use_prefill_window_in_sink_attention": false,
+        "window_length": null
+      },
+      "ffn": {
+        "ffn_mult": 1.0,
+        "no_op": false,
+        "replace_with_linear": false,
+        "sparsify": null
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": null,
+        "no_op": true,
+        "num_sink_tokens": null,
+        "replace_with_linear": false,
+        "sparsify": null,
+        "unshifted_sink": false,
+        "use_prefill_window_in_sink_attention": false,
+        "window_length": null
+      },
+      "ffn": {
+        "ffn_mult": 1.0,
+        "no_op": false,
+        "replace_with_linear": false,
+        "sparsify": null
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": null,
+        "no_op": true,
+        "num_sink_tokens": null,
+        "replace_with_linear": false,
+        "sparsify": null,
+        "unshifted_sink": false,
+        "use_prefill_window_in_sink_attention": false,
+        "window_length": null
+      },
+      "ffn": {
+        "ffn_mult": 1.3125,
+        "no_op": false,
+        "replace_with_linear": false,
+        "sparsify": null
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": null,
+        "no_op": true,
+        "num_sink_tokens": null,
+        "replace_with_linear": false,
+        "sparsify": null,
+        "unshifted_sink": false,
+        "use_prefill_window_in_sink_attention": false,
+        "window_length": null
+      },
+      "ffn": {
+        "ffn_mult": 1.0,
+        "no_op": false,
+        "replace_with_linear": false,
+        "sparsify": null
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": null,
+        "no_op": true,
+        "num_sink_tokens": null,
+        "replace_with_linear": false,
+        "sparsify": null,
+        "unshifted_sink": false,
+        "use_prefill_window_in_sink_attention": false,
+        "window_length": null
+      },
+      "ffn": {
+        "ffn_mult": 1.0,
+        "no_op": false,
+        "replace_with_linear": false,
+        "sparsify": null
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": null,
+        "no_op": true,
+        "num_sink_tokens": null,
+        "replace_with_linear": false,
+        "sparsify": null,
+        "unshifted_sink": false,
+        "use_prefill_window_in_sink_attention": false,
+        "window_length": null
+      },
+      "ffn": {
+        "ffn_mult": 1.0,
+        "no_op": false,
+        "replace_with_linear": false,
+        "sparsify": null
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": null,
+        "no_op": true,
+        "num_sink_tokens": null,
+        "replace_with_linear": false,
+        "sparsify": null,
+        "unshifted_sink": false,
+        "use_prefill_window_in_sink_attention": false,
+        "window_length": null
+      },
+      "ffn": {
+        "ffn_mult": 1.3125,
+        "no_op": false,
+        "replace_with_linear": false,
+        "sparsify": null
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": null,
+        "no_op": true,
+        "num_sink_tokens": null,
+        "replace_with_linear": false,
+        "sparsify": null,
+        "unshifted_sink": false,
+        "use_prefill_window_in_sink_attention": false,
+        "window_length": null
+      },
+      "ffn": {
+        "ffn_mult": 1.3125,
+        "no_op": false,
+        "replace_with_linear": false,
+        "sparsify": null
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": null,
+        "no_op": true,
+        "num_sink_tokens": null,
+        "replace_with_linear": false,
+        "sparsify": null,
+        "unshifted_sink": false,
+        "use_prefill_window_in_sink_attention": false,
+        "window_length": null
+      },
+      "ffn": {
+        "ffn_mult": 0.5,
+        "no_op": false,
+        "replace_with_linear": false,
+        "sparsify": null
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": null,
+        "no_op": true,
+        "num_sink_tokens": null,
+        "replace_with_linear": false,
+        "sparsify": null,
+        "unshifted_sink": false,
+        "use_prefill_window_in_sink_attention": false,
+        "window_length": null
+      },
+      "ffn": {
+        "ffn_mult": 0.5,
+        "no_op": false,
+        "replace_with_linear": false,
+        "sparsify": null
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": null,
+        "no_op": true,
+        "num_sink_tokens": null,
+        "replace_with_linear": false,
+        "sparsify": null,
+        "unshifted_sink": false,
+        "use_prefill_window_in_sink_attention": false,
+        "window_length": null
+      },
+      "ffn": {
+        "ffn_mult": 1.0,
+        "no_op": false,
+        "replace_with_linear": false,
+        "sparsify": null
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": null,
+        "no_op": true,
+        "num_sink_tokens": null,
+        "replace_with_linear": false,
+        "sparsify": null,
+        "unshifted_sink": false,
+        "use_prefill_window_in_sink_attention": false,
+        "window_length": null
+      },
+      "ffn": {
+        "ffn_mult": 1.0,
+        "no_op": false,
+        "replace_with_linear": false,
+        "sparsify": null
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": null,
+        "no_op": true,
+        "num_sink_tokens": null,
+        "replace_with_linear": false,
+        "sparsify": null,
+        "unshifted_sink": false,
+        "use_prefill_window_in_sink_attention": false,
+        "window_length": null
+      },
+      "ffn": {
+        "ffn_mult": 0.5,
+        "no_op": false,
+        "replace_with_linear": false,
+        "sparsify": null
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": null,
+        "no_op": true,
+        "num_sink_tokens": null,
+        "replace_with_linear": false,
+        "sparsify": null,
+        "unshifted_sink": false,
+        "use_prefill_window_in_sink_attention": false,
+        "window_length": null
+      },
+      "ffn": {
+        "ffn_mult": 0.5,
+        "no_op": false,
+        "replace_with_linear": false,
+        "sparsify": null
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": null,
+        "no_op": true,
+        "num_sink_tokens": null,
+        "replace_with_linear": false,
+        "sparsify": null,
+        "unshifted_sink": false,
+        "use_prefill_window_in_sink_attention": false,
+        "window_length": null
+      },
+      "ffn": {
+        "ffn_mult": 1.0,
+        "no_op": false,
+        "replace_with_linear": false,
+        "sparsify": null
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": null,
+        "no_op": true,
+        "num_sink_tokens": null,
+        "replace_with_linear": false,
+        "sparsify": null,
+        "unshifted_sink": false,
+        "use_prefill_window_in_sink_attention": false,
+        "window_length": null
+      },
+      "ffn": {
+        "ffn_mult": 0.5,
+        "no_op": false,
+        "replace_with_linear": false,
+        "sparsify": null
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": null,
+        "no_op": true,
+        "num_sink_tokens": null,
+        "replace_with_linear": false,
+        "sparsify": null,
+        "unshifted_sink": false,
+        "use_prefill_window_in_sink_attention": false,
+        "window_length": null
+      },
+      "ffn": {
+        "ffn_mult": 0.5,
+        "no_op": false,
+        "replace_with_linear": false,
+        "sparsify": null
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "num_sink_tokens": null,
+        "replace_with_linear": false,
+        "sparsify": null,
+        "unshifted_sink": false,
+        "use_prefill_window_in_sink_attention": false,
+        "window_length": null
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false,
+        "sparsify": null
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "num_sink_tokens": null,
+        "replace_with_linear": false,
+        "sparsify": null,
+        "unshifted_sink": false,
+        "use_prefill_window_in_sink_attention": false,
+        "window_length": null
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false,
+        "sparsify": null
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "num_sink_tokens": null,
+        "replace_with_linear": false,
+        "sparsify": null,
+        "unshifted_sink": false,
+        "use_prefill_window_in_sink_attention": false,
+        "window_length": null
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false,
+        "sparsify": null
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "num_sink_tokens": null,
+        "replace_with_linear": false,
+        "sparsify": null,
+        "unshifted_sink": false,
+        "use_prefill_window_in_sink_attention": false,
+        "window_length": null
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false,
+        "sparsify": null
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "num_sink_tokens": null,
+        "replace_with_linear": false,
+        "sparsify": null,
+        "unshifted_sink": false,
+        "use_prefill_window_in_sink_attention": false,
+        "window_length": null
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false,
+        "sparsify": null
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "num_sink_tokens": null,
+        "replace_with_linear": false,
+        "sparsify": null,
+        "unshifted_sink": false,
+        "use_prefill_window_in_sink_attention": false,
+        "window_length": null
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false,
+        "sparsify": null
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "num_sink_tokens": null,
+        "replace_with_linear": false,
+        "sparsify": null,
+        "unshifted_sink": false,
+        "use_prefill_window_in_sink_attention": false,
+        "window_length": null
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false,
+        "sparsify": null
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "num_sink_tokens": null,
+        "replace_with_linear": false,
+        "sparsify": null,
+        "unshifted_sink": false,
+        "use_prefill_window_in_sink_attention": false,
+        "window_length": null
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false,
+        "sparsify": null
+      }
+    },
+    {
+      "attention": {
+        "n_heads_in_group": 8,
+        "no_op": false,
+        "num_sink_tokens": null,
+        "replace_with_linear": false,
+        "sparsify": null,
+        "unshifted_sink": false,
+        "use_prefill_window_in_sink_attention": false,
+        "window_length": null
+      },
+      "ffn": {
+        "ffn_mult": 5.25,
+        "no_op": false,
+        "replace_with_linear": false,
+        "sparsify": null
+      }
+    }
+  ],
+  "bos_token_id": 128000,
+  "eos_token_id": 128009,
+  "hidden_act": "silu",
+  "hidden_size": 8192,
+  "initializer_range": 0.02,
+  "intermediate_size": null,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "nemotron-nas",
+  "num_attention_heads": 64,
+  "num_hidden_layers": 80,
+  "num_key_value_heads": null,
+  "pretraining_tp": 1,
+  "quantization_config": {
+    "_load_in_4bit": true,
+    "_load_in_8bit": false,
+    "bnb_4bit_compute_dtype": "bfloat16",
+    "bnb_4bit_quant_storage": "bfloat16",
+    "bnb_4bit_quant_type": "nf4",
+    "bnb_4bit_use_double_quant": true,
+    "llm_int8_enable_fp32_cpu_offload": false,
+    "llm_int8_has_fp16_weight": false,
+    "llm_int8_skip_modules": null,
+    "llm_int8_threshold": 6.0,
+    "load_in_4bit": true,
+    "load_in_8bit": false,
+    "quant_method": "bitsandbytes"
+  },
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": {
+    "factor": 8.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_type": "llama3"
+  },
+  "rope_theta": 500000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.49.0",
+  "use_cache": false,
+  "vocab_size": 128256
+}
diff --git a/special_tokens_map.json b/special_tokens_map.json
new file mode 100644
index 0000000000000000000000000000000000000000..278b7f0f84be865c4687700ee7b3c63d89a51e18
--- /dev/null
+++ b/special_tokens_map.json
@@ -0,0 +1,23 @@
+{
+  "bos_token": {
+    "content": "<|begin_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|eot_id|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|end_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}
diff --git a/tokenizer.json b/tokenizer.json
new file mode 100644
index 0000000000000000000000000000000000000000..1c1d8d5c9024994f1d3b00f9662b8dd89ca13cf2
--- /dev/null
+++ b/tokenizer.json
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920
diff --git a/tokenizer_config.json b/tokenizer_config.json
new file mode 100644
index 0000000000000000000000000000000000000000..edd01b980c1db496ea102a51c972ee8f5d1a2c74
--- /dev/null
+++ b/tokenizer_config.json
@@ -0,0 +1,2064 @@
+{
+  "added_tokens_decoder": {
+    "128000": {
+      "content": "<|begin_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128001": {
+      "content": "<|end_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128002": {
+      "content": "<|reserved_special_token_0|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128003": {
+      "content": "<|reserved_special_token_1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128004": {
+      "content": "<|finetune_right_pad_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128005": {
+      "content": "<|reserved_special_token_2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128006": {
+      "content": "<|start_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128007": {
+      "content": "<|end_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128008": {
+      "content": "<|eom_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128009": {
+      "content": "<|eot_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128010": {
+      "content": "<|python_tag|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128011": {
+      "content": "<|reserved_special_token_3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128012": {
+      "content": "<|reserved_special_token_4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128013": {
+      "content": "<|reserved_special_token_5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128014": {
+      "content": "<|reserved_special_token_6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128015": {
+      "content": "<|reserved_special_token_7|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128016": {
+      "content": "<|reserved_special_token_8|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128017": {
+      "content": "<|reserved_special_token_9|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128018": {
+      "content": "<|reserved_special_token_10|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128019": {
+      "content": "<|reserved_special_token_11|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128020": {
+      "content": "<|reserved_special_token_12|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128021": {
+      "content": "<|reserved_special_token_13|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128022": {
+      "content": "<|reserved_special_token_14|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128023": {
+      "content": "<|reserved_special_token_15|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128024": {
+      "content": "<|reserved_special_token_16|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128025": {
+      "content": "<|reserved_special_token_17|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128026": {
+      "content": "<|reserved_special_token_18|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128027": {
+      "content": "<|reserved_special_token_19|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128028": {
+      "content": "<|reserved_special_token_20|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128029": {
+      "content": "<|reserved_special_token_21|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128030": {
+      "content": "<|reserved_special_token_22|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128031": {
+      "content": "<|reserved_special_token_23|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128032": {
+      "content": "<|reserved_special_token_24|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128033": {
+      "content": "<|reserved_special_token_25|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128034": {
+      "content": "<|reserved_special_token_26|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128035": {
+      "content": "<|reserved_special_token_27|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128036": {
+      "content": "<|reserved_special_token_28|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128037": {
+      "content": "<|reserved_special_token_29|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128038": {
+      "content": "<|reserved_special_token_30|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128039": {
+      "content": "<|reserved_special_token_31|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128040": {
+      "content": "<|reserved_special_token_32|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128041": {
+      "content": "<|reserved_special_token_33|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128042": {
+      "content": "<|reserved_special_token_34|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128043": {
+      "content": "<|reserved_special_token_35|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128044": {
+      "content": "<|reserved_special_token_36|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128045": {
+      "content": "<|reserved_special_token_37|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128046": {
+      "content": "<|reserved_special_token_38|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128047": {
+      "content": "<|reserved_special_token_39|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128048": {
+      "content": "<|reserved_special_token_40|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128049": {
+      "content": "<|reserved_special_token_41|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128050": {
+      "content": "<|reserved_special_token_42|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128051": {
+      "content": "<|reserved_special_token_43|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128052": {
+      "content": "<|reserved_special_token_44|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128053": {
+      "content": "<|reserved_special_token_45|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128054": {
+      "content": "<|reserved_special_token_46|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128055": {
+      "content": "<|reserved_special_token_47|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128056": {
+      "content": "<|reserved_special_token_48|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128057": {
+      "content": "<|reserved_special_token_49|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128058": {
+      "content": "<|reserved_special_token_50|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128059": {
+      "content": "<|reserved_special_token_51|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128060": {
+      "content": "<|reserved_special_token_52|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128061": {
+      "content": "<|reserved_special_token_53|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128062": {
+      "content": "<|reserved_special_token_54|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128063": {
+      "content": "<|reserved_special_token_55|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128064": {
+      "content": "<|reserved_special_token_56|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128065": {
+      "content": "<|reserved_special_token_57|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128066": {
+      "content": "<|reserved_special_token_58|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128067": {
+      "content": "<|reserved_special_token_59|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128068": {
+      "content": "<|reserved_special_token_60|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128069": {
+      "content": "<|reserved_special_token_61|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128070": {
+      "content": "<|reserved_special_token_62|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128071": {
+      "content": "<|reserved_special_token_63|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128072": {
+      "content": "<|reserved_special_token_64|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128073": {
+      "content": "<|reserved_special_token_65|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128074": {
+      "content": "<|reserved_special_token_66|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128075": {
+      "content": "<|reserved_special_token_67|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128076": {
+      "content": "<|reserved_special_token_68|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128077": {
+      "content": "<|reserved_special_token_69|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128078": {
+      "content": "<|reserved_special_token_70|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128079": {
+      "content": "<|reserved_special_token_71|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128080": {
+      "content": "<|reserved_special_token_72|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128081": {
+      "content": "<|reserved_special_token_73|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128082": {
+      "content": "<|reserved_special_token_74|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128083": {
+      "content": "<|reserved_special_token_75|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128084": {
+      "content": "<|reserved_special_token_76|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128085": {
+      "content": "<|reserved_special_token_77|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128086": {
+      "content": "<|reserved_special_token_78|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128087": {
+      "content": "<|reserved_special_token_79|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128088": {
+      "content": "<|reserved_special_token_80|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128089": {
+      "content": "<|reserved_special_token_81|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128090": {
+      "content": "<|reserved_special_token_82|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128091": {
+      "content": "<|reserved_special_token_83|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128092": {
+      "content": "<|reserved_special_token_84|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128093": {
+      "content": "<|reserved_special_token_85|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128094": {
+      "content": "<|reserved_special_token_86|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128095": {
+      "content": "<|reserved_special_token_87|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128096": {
+      "content": "<|reserved_special_token_88|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128097": {
+      "content": "<|reserved_special_token_89|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128098": {
+      "content": "<|reserved_special_token_90|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128099": {
+      "content": "<|reserved_special_token_91|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128100": {
+      "content": "<|reserved_special_token_92|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128101": {
+      "content": "<|reserved_special_token_93|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128102": {
+      "content": "<|reserved_special_token_94|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128103": {
+      "content": "<|reserved_special_token_95|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128104": {
+      "content": "<|reserved_special_token_96|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128105": {
+      "content": "<|reserved_special_token_97|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128106": {
+      "content": "<|reserved_special_token_98|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128107": {
+      "content": "<|reserved_special_token_99|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128108": {
+      "content": "<|reserved_special_token_100|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128109": {
+      "content": "<|reserved_special_token_101|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128110": {
+      "content": "<|reserved_special_token_102|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128111": {
+      "content": "<|reserved_special_token_103|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128112": {
+      "content": "<|reserved_special_token_104|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128113": {
+      "content": "<|reserved_special_token_105|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128114": {
+      "content": "<|reserved_special_token_106|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128115": {
+      "content": "<|reserved_special_token_107|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128116": {
+      "content": "<|reserved_special_token_108|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128117": {
+      "content": "<|reserved_special_token_109|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128118": {
+      "content": "<|reserved_special_token_110|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128119": {
+      "content": "<|reserved_special_token_111|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128120": {
+      "content": "<|reserved_special_token_112|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128121": {
+      "content": "<|reserved_special_token_113|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128122": {
+      "content": "<|reserved_special_token_114|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128123": {
+      "content": "<|reserved_special_token_115|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128124": {
+      "content": "<|reserved_special_token_116|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128125": {
+      "content": "<|reserved_special_token_117|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128126": {
+      "content": "<|reserved_special_token_118|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128127": {
+      "content": "<|reserved_special_token_119|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128128": {
+      "content": "<|reserved_special_token_120|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128129": {
+      "content": "<|reserved_special_token_121|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128130": {
+      "content": "<|reserved_special_token_122|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128131": {
+      "content": "<|reserved_special_token_123|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128132": {
+      "content": "<|reserved_special_token_124|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128133": {
+      "content": "<|reserved_special_token_125|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128134": {
+      "content": "<|reserved_special_token_126|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128135": {
+      "content": "<|reserved_special_token_127|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128136": {
+      "content": "<|reserved_special_token_128|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128137": {
+      "content": "<|reserved_special_token_129|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128138": {
+      "content": "<|reserved_special_token_130|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128139": {
+      "content": "<|reserved_special_token_131|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128140": {
+      "content": "<|reserved_special_token_132|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128141": {
+      "content": "<|reserved_special_token_133|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128142": {
+      "content": "<|reserved_special_token_134|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128143": {
+      "content": "<|reserved_special_token_135|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128144": {
+      "content": "<|reserved_special_token_136|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128145": {
+      "content": "<|reserved_special_token_137|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128146": {
+      "content": "<|reserved_special_token_138|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128147": {
+      "content": "<|reserved_special_token_139|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128148": {
+      "content": "<|reserved_special_token_140|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128149": {
+      "content": "<|reserved_special_token_141|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128150": {
+      "content": "<|reserved_special_token_142|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128151": {
+      "content": "<|reserved_special_token_143|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128152": {
+      "content": "<|reserved_special_token_144|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128153": {
+      "content": "<|reserved_special_token_145|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128154": {
+      "content": "<|reserved_special_token_146|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128155": {
+      "content": "<|reserved_special_token_147|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128156": {
+      "content": "<|reserved_special_token_148|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128157": {
+      "content": "<|reserved_special_token_149|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128158": {
+      "content": "<|reserved_special_token_150|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128159": {
+      "content": "<|reserved_special_token_151|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128160": {
+      "content": "<|reserved_special_token_152|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128161": {
+      "content": "<|reserved_special_token_153|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128162": {
+      "content": "<|reserved_special_token_154|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128163": {
+      "content": "<|reserved_special_token_155|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128164": {
+      "content": "<|reserved_special_token_156|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128165": {
+      "content": "<|reserved_special_token_157|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128166": {
+      "content": "<|reserved_special_token_158|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128167": {
+      "content": "<|reserved_special_token_159|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128168": {
+      "content": "<|reserved_special_token_160|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128169": {
+      "content": "<|reserved_special_token_161|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128170": {
+      "content": "<|reserved_special_token_162|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128171": {
+      "content": "<|reserved_special_token_163|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128172": {
+      "content": "<|reserved_special_token_164|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128173": {
+      "content": "<|reserved_special_token_165|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128174": {
+      "content": "<|reserved_special_token_166|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128175": {
+      "content": "<|reserved_special_token_167|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128176": {
+      "content": "<|reserved_special_token_168|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128177": {
+      "content": "<|reserved_special_token_169|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128178": {
+      "content": "<|reserved_special_token_170|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128179": {
+      "content": "<|reserved_special_token_171|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128180": {
+      "content": "<|reserved_special_token_172|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128181": {
+      "content": "<|reserved_special_token_173|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128182": {
+      "content": "<|reserved_special_token_174|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128183": {
+      "content": "<|reserved_special_token_175|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128184": {
+      "content": "<|reserved_special_token_176|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128185": {
+      "content": "<|reserved_special_token_177|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128186": {
+      "content": "<|reserved_special_token_178|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128187": {
+      "content": "<|reserved_special_token_179|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128188": {
+      "content": "<|reserved_special_token_180|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128189": {
+      "content": "<|reserved_special_token_181|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128190": {
+      "content": "<|reserved_special_token_182|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128191": {
+      "content": "<|reserved_special_token_183|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128192": {
+      "content": "<|reserved_special_token_184|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128193": {
+      "content": "<|reserved_special_token_185|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128194": {
+      "content": "<|reserved_special_token_186|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128195": {
+      "content": "<|reserved_special_token_187|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128196": {
+      "content": "<|reserved_special_token_188|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128197": {
+      "content": "<|reserved_special_token_189|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128198": {
+      "content": "<|reserved_special_token_190|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128199": {
+      "content": "<|reserved_special_token_191|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128200": {
+      "content": "<|reserved_special_token_192|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128201": {
+      "content": "<|reserved_special_token_193|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128202": {
+      "content": "<|reserved_special_token_194|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128203": {
+      "content": "<|reserved_special_token_195|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128204": {
+      "content": "<|reserved_special_token_196|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128205": {
+      "content": "<|reserved_special_token_197|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128206": {
+      "content": "<|reserved_special_token_198|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128207": {
+      "content": "<|reserved_special_token_199|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128208": {
+      "content": "<|reserved_special_token_200|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128209": {
+      "content": "<|reserved_special_token_201|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128210": {
+      "content": "<|reserved_special_token_202|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128211": {
+      "content": "<|reserved_special_token_203|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128212": {
+      "content": "<|reserved_special_token_204|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128213": {
+      "content": "<|reserved_special_token_205|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128214": {
+      "content": "<|reserved_special_token_206|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128215": {
+      "content": "<|reserved_special_token_207|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128216": {
+      "content": "<|reserved_special_token_208|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128217": {
+      "content": "<|reserved_special_token_209|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128218": {
+      "content": "<|reserved_special_token_210|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128219": {
+      "content": "<|reserved_special_token_211|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128220": {
+      "content": "<|reserved_special_token_212|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128221": {
+      "content": "<|reserved_special_token_213|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128222": {
+      "content": "<|reserved_special_token_214|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128223": {
+      "content": "<|reserved_special_token_215|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128224": {
+      "content": "<|reserved_special_token_216|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128225": {
+      "content": "<|reserved_special_token_217|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128226": {
+      "content": "<|reserved_special_token_218|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128227": {
+      "content": "<|reserved_special_token_219|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128228": {
+      "content": "<|reserved_special_token_220|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128229": {
+      "content": "<|reserved_special_token_221|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128230": {
+      "content": "<|reserved_special_token_222|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128231": {
+      "content": "<|reserved_special_token_223|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128232": {
+      "content": "<|reserved_special_token_224|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128233": {
+      "content": "<|reserved_special_token_225|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128234": {
+      "content": "<|reserved_special_token_226|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128235": {
+      "content": "<|reserved_special_token_227|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128236": {
+      "content": "<|reserved_special_token_228|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128237": {
+      "content": "<|reserved_special_token_229|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128238": {
+      "content": "<|reserved_special_token_230|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128239": {
+      "content": "<|reserved_special_token_231|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128240": {
+      "content": "<|reserved_special_token_232|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128241": {
+      "content": "<|reserved_special_token_233|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128242": {
+      "content": "<|reserved_special_token_234|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128243": {
+      "content": "<|reserved_special_token_235|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128244": {
+      "content": "<|reserved_special_token_236|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128245": {
+      "content": "<|reserved_special_token_237|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128246": {
+      "content": "<|reserved_special_token_238|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128247": {
+      "content": "<|reserved_special_token_239|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128248": {
+      "content": "<|reserved_special_token_240|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128249": {
+      "content": "<|reserved_special_token_241|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128250": {
+      "content": "<|reserved_special_token_242|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128251": {
+      "content": "<|reserved_special_token_243|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128252": {
+      "content": "<|reserved_special_token_244|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128253": {
+      "content": "<|reserved_special_token_245|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128254": {
+      "content": "<|reserved_special_token_246|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128255": {
+      "content": "<|reserved_special_token_247|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|begin_of_text|>",
+  "chat_template": "{{- bos_token }}{%- if messages[0]['role'] == 'system' %}{%- set system_message = messages[0]['content']|trim %}{%- set messages = messages[1:] %}{%- else %}{%- set system_message = \"\" %}{%- endif %}{{- \"<|start_header_id|>system<|end_header_id|>\\n\\n\" }}{{- system_message }}{{- \"<|eot_id|>\" }}{%- for message in messages %}{%- if message['role'] == 'assistant' and '</think>' in message['content'] %}{%- set content = message['content'].split('</think>')[-1].lstrip() %}{%- else %}{%- set content = message['content'] %}{%- endif %}{{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\\n\\n' + content | trim + '<|eot_id|>' }}{%- endfor %}{%- if add_generation_prompt %}{{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' }}{%- endif %}",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|eot_id|>",
+  "extra_special_tokens": {},
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "pad_token": "<|end_of_text|>",
+  "tokenizer_class": "PreTrainedTokenizer"
+}