Initial upload of fine-tuned model

.gitattributes

@@ -33,3 +33,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+checkpoint-3670/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoint-7340/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,58 @@
+---
+base_model: Maxtimer97/Llama2SWA
+library_name: transformers
+model_name: model
+tags:
+- generated_from_trainer
+- trl
+- sft
+licence: license
+---
+
+# Model Card for model
+
+This model is a fine-tuned version of [Maxtimer97/Llama2SWA](https://huggingface.co/Maxtimer97/Llama2SWA).
+It has been trained using [TRL](https://github.com/huggingface/trl).
+
+## Quick start
+
+```python
+from transformers import pipeline
+
+question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?"
+generator = pipeline("text-generation", model="None", device="cuda")
+output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0]
+print(output["generated_text"])
+```
+
+## Training procedure
+
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/maxtimer/huggingface/runs/yqhjmjuu) 
+
+
+This model was trained with SFT.
+
+### Framework versions
+
+- TRL: 0.19.1
+- Transformers: 4.53.0
+- Pytorch: 2.6.0
+- Datasets: 3.6.0
+- Tokenizers: 0.21.1
+
+## Citations
+
+
+
+Cite TRL as:
+    
+```bibtex
+@misc{vonwerra2022trl,
+	title        = {{TRL: Transformer Reinforcement Learning}},
+	author       = {Leandro von Werra and Younes Belkada and Lewis Tunstall and Edward Beeching and Tristan Thrush and Nathan Lambert and Shengyi Huang and Kashif Rasul and Quentin Gallou{\'e}dec},
+	year         = 2020,
+	journal      = {GitHub repository},
+	publisher    = {GitHub},
+	howpublished = {\url{https://github.com/huggingface/trl}}
+}
+```

chat_template.jinja

@@ -0,0 +1,21 @@
+
+    {{- bos_token }}
+    {%- for message in messages %}
+        {%- if message['role'] == 'assistant' %}
+            {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>' + '
+' }}
+            {%- generation %}
+            {{- message['content'] + '<|eot_id|>' }}
+            {%- endgeneration %}
+            {{- '
+' }}
+        {%- else %}
+            {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>' + '
+' + message['content'] + '<|eot_id|>' + '
+' }}
+        {%- endif %}
+    {%- endfor %}
+    {%- if add_generation_prompt %}
+        {{- '<|start_header_id|>assistant<|end_header_id|>
+' }}
+    {%- endif %}

checkpoint-3670/chat_template.jinja

@@ -0,0 +1,21 @@
+
+    {{- bos_token }}
+    {%- for message in messages %}
+        {%- if message['role'] == 'assistant' %}
+            {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>' + '
+' }}
+            {%- generation %}
+            {{- message['content'] + '<|eot_id|>' }}
+            {%- endgeneration %}
+            {{- '
+' }}
+        {%- else %}
+            {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>' + '
+' + message['content'] + '<|eot_id|>' + '
+' }}
+        {%- endif %}
+    {%- endfor %}
+    {%- if add_generation_prompt %}
+        {{- '<|start_header_id|>assistant<|end_header_id|>
+' }}
+    {%- endif %}

checkpoint-3670/config.json

@@ -0,0 +1,130 @@
+{
+  "architectures": [
+    "HymbaForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "attn_hidden_size": -1,
+  "attn_implementation": "flash_attention_2",
+  "attn_implementation_new": "flash_attention_2",
+  "attn_only_wo_proj": true,
+  "auto_map": {
+    "AutoConfig": "configuration_hymba.HymbaConfig",
+    "AutoModelForCausalLM": "modeling_hymba.HymbaForCausalLM"
+  },
+  "bos_token_id": 128000,
+  "calc_logits_for_entire_prompt": false,
+  "conv_dim": {
+    "0": 3200,
+    "1": 3200,
+    "10": 3200,
+    "11": 3200,
+    "12": 3200,
+    "13": 3200,
+    "14": 3200,
+    "15": 3200,
+    "16": 3200,
+    "17": 3200,
+    "18": 3200,
+    "19": 3200,
+    "2": 3200,
+    "20": 3200,
+    "21": 3200,
+    "22": 3200,
+    "23": 3200,
+    "24": 3200,
+    "25": 3200,
+    "26": 3200,
+    "27": 3200,
+    "28": 3200,
+    "29": 3200,
+    "3": 3200,
+    "30": 3200,
+    "31": 3200,
+    "4": 3200,
+    "5": 3200,
+    "6": 3200,
+    "7": 3200,
+    "8": 3200,
+    "9": 3200
+  },
+  "eos_token_id": [
+    128001,
+    128008,
+    128009
+  ],
+  "global_attn_idx": [
+    0,
+    8,
+    15
+  ],
+  "hidden_act": "silu",
+  "hidden_size": 2048,
+  "initializer_range": 0.02,
+  "intermediate_size": 8192,
+  "kq_head_dim": 64,
+  "kq_norm": "none",
+  "kv_reuse_every_i_layer": -1,
+  "kv_reuse_group": null,
+  "kv_weight_reuse": false,
+  "layer_type": [
+    "h",
+    "h",
+    "h",
+    "h",
+    "h",
+    "h",
+    "h",
+    "h",
+    "h",
+    "h",
+    "h",
+    "h",
+    "h",
+    "h",
+    "h",
+    "h"
+  ],
+  "mamba_conv_bias": true,
+  "mamba_d_conv": 4,
+  "mamba_d_state": 16,
+  "mamba_dt_rank": 100,
+  "mamba_expand": 1,
+  "mamba_inner_layernorms": true,
+  "mamba_proj_bias": false,
+  "max_position_embeddings": 131072,
+  "memory_tokens_interspersed_every": 0,
+  "mlp_hidden_act": "silu",
+  "model_type": "hymba",
+  "num_attention_heads": 32,
+  "num_experts": 1,
+  "num_experts_per_tok": 1,
+  "num_hidden_layers": 16,
+  "num_key_value_heads": 8,
+  "num_mamba": 1,
+  "num_memory_tokens": 0,
+  "orig_max_position_embeddings": 2048,
+  "output_router_logits": false,
+  "pad_token_id": null,
+  "pure_attn": true,
+  "rms_norm_eps": 1e-05,
+  "rope": true,
+  "rope_scaling": {
+    "factor": 32.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_type": "llama3"
+  },
+  "rope_theta": 500000.0,
+  "rope_type": "ntk",
+  "router_aux_loss_coef": 0.001,
+  "seq_length": 8192,
+  "sliding_window": 1024,
+  "tie_word_embeddings": true,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.53.0",
+  "use_cache": false,
+  "use_mamba_kernels": true,
+  "v_head_dim": 64,
+  "vocab_size": 128256
+}

checkpoint-3670/configuration_hymba.py

@@ -0,0 +1,120 @@
+import math
+from transformers.configuration_utils import PretrainedConfig
+
+
+class HymbaConfig(PretrainedConfig):
+
+    model_type = "hymba"
+    keys_to_ignore_at_inference = ["past_key_values"]
+
+    def __init__(
+            self,
+            vocab_size=65536,
+            tie_word_embeddings=False,
+            hidden_size=4096,
+            intermediate_size=14336,
+            num_hidden_layers=32,
+            num_attention_heads=32,
+            num_key_value_heads=8,
+            hidden_act="silu",
+            initializer_range=0.02,
+            rms_norm_eps=1e-6,
+            attn_only_wo_proj=False,
+            use_cache=True,
+            calc_logits_for_entire_prompt=False,
+            output_router_logits=False,
+            router_aux_loss_coef=0.001,
+            pad_token_id=0,
+            bos_token_id=1,
+            eos_token_id=2,
+            sliding_window=None,
+            max_position_embeddings=262144,
+            orig_max_position_embeddings=None,
+            attention_dropout=0.0,
+            num_experts_per_tok=2,
+            num_experts=16,
+            use_mamba_kernels=True,
+            mamba_d_state=16,
+            mamba_d_conv=4,
+            mamba_expand=2,
+            mamba_dt_rank="auto",
+            mamba_conv_bias=True,
+            mamba_proj_bias=False,
+            mamba_inner_layernorms=True,
+            kv_reuse_every_i_layer=-1,
+            kv_reuse_group=None,
+            kv_weight_reuse=False,
+            global_attn_idx=None,
+            num_mamba=1,
+            pure_attn=False,
+            attn_implementation_new='sdpa',
+            rope_type=None,
+            **kwargs,
+    ):
+        self.vocab_size = vocab_size
+        self.tie_word_embeddings = tie_word_embeddings
+        self.hidden_size = hidden_size
+        self.intermediate_size = intermediate_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+        self.sliding_window = sliding_window
+        self.max_position_embeddings = max_position_embeddings
+        self.orig_max_position_embeddings = orig_max_position_embeddings
+        self.attention_dropout = attention_dropout
+        self.attn_only_wo_proj = attn_only_wo_proj
+
+        if num_key_value_heads is None:
+            num_key_value_heads = num_attention_heads
+
+        self.num_key_value_heads = num_key_value_heads
+        self.hidden_act = hidden_act
+        self.initializer_range = initializer_range
+        self.rms_norm_eps = rms_norm_eps
+
+        self.use_cache = use_cache
+        self.calc_logits_for_entire_prompt = calc_logits_for_entire_prompt
+        self.output_router_logits = output_router_logits
+        self.router_aux_loss_coef = router_aux_loss_coef
+
+        self.num_experts_per_tok = num_experts_per_tok
+        self.num_experts = num_experts
+
+        self.use_mamba_kernels = use_mamba_kernels
+        self.mamba_d_state = mamba_d_state
+        self.mamba_d_conv = mamba_d_conv
+        self.mamba_expand = mamba_expand
+        self.mamba_dt_rank = math.ceil(self.hidden_size / 16) if mamba_dt_rank == "auto" else mamba_dt_rank
+        self.mamba_conv_bias = mamba_conv_bias
+        self.mamba_proj_bias = mamba_proj_bias
+        self.mamba_inner_layernorms = mamba_inner_layernorms
+
+        self.attn_hidden_size = kwargs.pop("attn_hidden_size", -1)
+        self.kq_head_dim = kwargs.pop("kq_head_dim", -1)
+        self.v_head_dim = kwargs.pop("v_head_dim", -1)
+        self.kq_norm = kwargs.pop("kq_norm", None)
+        self.rope = kwargs.pop("rope", False)
+        self.rope_theta = kwargs.pop("rope_theta", 10000.0)
+        self.num_memory_tokens = kwargs.pop("num_memory_tokens", 0)
+        self.memory_tokens_interspersed_every = kwargs.pop("memory_tokens_interspersed_every", 0)
+
+        self.kv_reuse_every_i_layer = kv_reuse_every_i_layer
+        self.kv_reuse_group = kv_reuse_group
+        self.kv_weight_reuse = kv_weight_reuse
+
+        self.global_attn_idx = global_attn_idx
+
+        self.num_mamba = num_mamba
+
+        self.attn_implementation_new = attn_implementation_new
+
+        self.rope_type = rope_type
+
+        self.pure_attn = pure_attn
+        
+        super().__init__(
+            pad_token_id=pad_token_id,
+            bos_token_id=bos_token_id,
+            eos_token_id=eos_token_id,
+            tie_word_embeddings=tie_word_embeddings,
+            **kwargs,
+        )

checkpoint-3670/generation_config.json

@@ -0,0 +1,10 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 128000,
+  "eos_token_id": [
+    128001,
+    128008,
+    128009
+  ],
+  "transformers_version": "4.53.0"
+}

checkpoint-3670/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:67b1bfac6c8161e1dce2baf94bb68c835ec0f35176b74c2784cab465640d7c64
+size 2471777104

checkpoint-3670/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b0dbe755c16407d32c986de9597bfbce7d29c4a1f42cecb4652aad6686b12684
+size 4943354486

checkpoint-3670/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:386fcc8cc1089aade9450d86fb239ea3483f455fd2d78d8378645feecfec9d69
+size 14244

checkpoint-3670/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d4846267e0a17ad491353c9e36f2e436fbe4fe2b2cfcb7043301852b07efd9cf
+size 1064

checkpoint-3670/special_tokens_map.json

@@ -0,0 +1,17 @@
+{
+  "bos_token": {
+    "content": "<|begin_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|eot_id|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<|eot_id|>"
+}

checkpoint-3670/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920

checkpoint-3670/tokenizer_config.json

@@ -0,0 +1,2063 @@
+{
+  "added_tokens_decoder": {
+    "128000": {
+      "content": "<|begin_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128001": {
+      "content": "<|end_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128002": {
+      "content": "<|reserved_special_token_0|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128003": {
+      "content": "<|reserved_special_token_1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128004": {
+      "content": "<|finetune_right_pad_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128005": {
+      "content": "<|reserved_special_token_2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128006": {
+      "content": "<|start_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128007": {
+      "content": "<|end_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128008": {
+      "content": "<|eom_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128009": {
+      "content": "<|eot_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128010": {
+      "content": "<|python_tag|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128011": {
+      "content": "<|reserved_special_token_3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128012": {
+      "content": "<|reserved_special_token_4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128013": {
+      "content": "<|reserved_special_token_5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128014": {
+      "content": "<|reserved_special_token_6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128015": {
+      "content": "<|reserved_special_token_7|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128016": {
+      "content": "<|reserved_special_token_8|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128017": {
+      "content": "<|reserved_special_token_9|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128018": {
+      "content": "<|reserved_special_token_10|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128019": {
+      "content": "<|reserved_special_token_11|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128020": {
+      "content": "<|reserved_special_token_12|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128021": {
+      "content": "<|reserved_special_token_13|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128022": {
+      "content": "<|reserved_special_token_14|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128023": {
+      "content": "<|reserved_special_token_15|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128024": {
+      "content": "<|reserved_special_token_16|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128025": {
+      "content": "<|reserved_special_token_17|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128026": {
+      "content": "<|reserved_special_token_18|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128027": {
+      "content": "<|reserved_special_token_19|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128028": {
+      "content": "<|reserved_special_token_20|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128029": {
+      "content": "<|reserved_special_token_21|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128030": {
+      "content": "<|reserved_special_token_22|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128031": {
+      "content": "<|reserved_special_token_23|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128032": {
+      "content": "<|reserved_special_token_24|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128033": {
+      "content": "<|reserved_special_token_25|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128034": {
+      "content": "<|reserved_special_token_26|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128035": {
+      "content": "<|reserved_special_token_27|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128036": {
+      "content": "<|reserved_special_token_28|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128037": {
+      "content": "<|reserved_special_token_29|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128038": {
+      "content": "<|reserved_special_token_30|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128039": {
+      "content": "<|reserved_special_token_31|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128040": {
+      "content": "<|reserved_special_token_32|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128041": {
+      "content": "<|reserved_special_token_33|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128042": {
+      "content": "<|reserved_special_token_34|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128043": {
+      "content": "<|reserved_special_token_35|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128044": {
+      "content": "<|reserved_special_token_36|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128045": {
+      "content": "<|reserved_special_token_37|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128046": {
+      "content": "<|reserved_special_token_38|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128047": {
+      "content": "<|reserved_special_token_39|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128048": {
+      "content": "<|reserved_special_token_40|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128049": {
+      "content": "<|reserved_special_token_41|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128050": {
+      "content": "<|reserved_special_token_42|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128051": {
+      "content": "<|reserved_special_token_43|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128052": {
+      "content": "<|reserved_special_token_44|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128053": {
+      "content": "<|reserved_special_token_45|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128054": {
+      "content": "<|reserved_special_token_46|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128055": {
+      "content": "<|reserved_special_token_47|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128056": {
+      "content": "<|reserved_special_token_48|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128057": {
+      "content": "<|reserved_special_token_49|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128058": {
+      "content": "<|reserved_special_token_50|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128059": {
+      "content": "<|reserved_special_token_51|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128060": {
+      "content": "<|reserved_special_token_52|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128061": {
+      "content": "<|reserved_special_token_53|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128062": {
+      "content": "<|reserved_special_token_54|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128063": {
+      "content": "<|reserved_special_token_55|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128064": {
+      "content": "<|reserved_special_token_56|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128065": {
+      "content": "<|reserved_special_token_57|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128066": {
+      "content": "<|reserved_special_token_58|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128067": {
+      "content": "<|reserved_special_token_59|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128068": {
+      "content": "<|reserved_special_token_60|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128069": {
+      "content": "<|reserved_special_token_61|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128070": {
+      "content": "<|reserved_special_token_62|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128071": {
+      "content": "<|reserved_special_token_63|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128072": {
+      "content": "<|reserved_special_token_64|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128073": {
+      "content": "<|reserved_special_token_65|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128074": {
+      "content": "<|reserved_special_token_66|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128075": {
+      "content": "<|reserved_special_token_67|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128076": {
+      "content": "<|reserved_special_token_68|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128077": {
+      "content": "<|reserved_special_token_69|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128078": {
+      "content": "<|reserved_special_token_70|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128079": {
+      "content": "<|reserved_special_token_71|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128080": {
+      "content": "<|reserved_special_token_72|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128081": {
+      "content": "<|reserved_special_token_73|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128082": {
+      "content": "<|reserved_special_token_74|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128083": {
+      "content": "<|reserved_special_token_75|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128084": {
+      "content": "<|reserved_special_token_76|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128085": {
+      "content": "<|reserved_special_token_77|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128086": {
+      "content": "<|reserved_special_token_78|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128087": {
+      "content": "<|reserved_special_token_79|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128088": {
+      "content": "<|reserved_special_token_80|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128089": {
+      "content": "<|reserved_special_token_81|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128090": {
+      "content": "<|reserved_special_token_82|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128091": {
+      "content": "<|reserved_special_token_83|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128092": {
+      "content": "<|reserved_special_token_84|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128093": {
+      "content": "<|reserved_special_token_85|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128094": {
+      "content": "<|reserved_special_token_86|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128095": {
+      "content": "<|reserved_special_token_87|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128096": {
+      "content": "<|reserved_special_token_88|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128097": {
+      "content": "<|reserved_special_token_89|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128098": {
+      "content": "<|reserved_special_token_90|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128099": {
+      "content": "<|reserved_special_token_91|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128100": {
+      "content": "<|reserved_special_token_92|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128101": {
+      "content": "<|reserved_special_token_93|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128102": {
+      "content": "<|reserved_special_token_94|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128103": {
+      "content": "<|reserved_special_token_95|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128104": {
+      "content": "<|reserved_special_token_96|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128105": {
+      "content": "<|reserved_special_token_97|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128106": {
+      "content": "<|reserved_special_token_98|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128107": {
+      "content": "<|reserved_special_token_99|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128108": {
+      "content": "<|reserved_special_token_100|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128109": {
+      "content": "<|reserved_special_token_101|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128110": {
+      "content": "<|reserved_special_token_102|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128111": {
+      "content": "<|reserved_special_token_103|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128112": {
+      "content": "<|reserved_special_token_104|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128113": {
+      "content": "<|reserved_special_token_105|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128114": {
+      "content": "<|reserved_special_token_106|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128115": {
+      "content": "<|reserved_special_token_107|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128116": {
+      "content": "<|reserved_special_token_108|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128117": {
+      "content": "<|reserved_special_token_109|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128118": {
+      "content": "<|reserved_special_token_110|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128119": {
+      "content": "<|reserved_special_token_111|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128120": {
+      "content": "<|reserved_special_token_112|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128121": {
+      "content": "<|reserved_special_token_113|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128122": {
+      "content": "<|reserved_special_token_114|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128123": {
+      "content": "<|reserved_special_token_115|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128124": {
+      "content": "<|reserved_special_token_116|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128125": {
+      "content": "<|reserved_special_token_117|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128126": {
+      "content": "<|reserved_special_token_118|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128127": {
+      "content": "<|reserved_special_token_119|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128128": {
+      "content": "<|reserved_special_token_120|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128129": {
+      "content": "<|reserved_special_token_121|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128130": {
+      "content": "<|reserved_special_token_122|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128131": {
+      "content": "<|reserved_special_token_123|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128132": {
+      "content": "<|reserved_special_token_124|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128133": {
+      "content": "<|reserved_special_token_125|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128134": {
+      "content": "<|reserved_special_token_126|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128135": {
+      "content": "<|reserved_special_token_127|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128136": {
+      "content": "<|reserved_special_token_128|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128137": {
+      "content": "<|reserved_special_token_129|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128138": {
+      "content": "<|reserved_special_token_130|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128139": {
+      "content": "<|reserved_special_token_131|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128140": {
+      "content": "<|reserved_special_token_132|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128141": {
+      "content": "<|reserved_special_token_133|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128142": {
+      "content": "<|reserved_special_token_134|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128143": {
+      "content": "<|reserved_special_token_135|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128144": {
+      "content": "<|reserved_special_token_136|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128145": {
+      "content": "<|reserved_special_token_137|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128146": {
+      "content": "<|reserved_special_token_138|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128147": {
+      "content": "<|reserved_special_token_139|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128148": {
+      "content": "<|reserved_special_token_140|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128149": {
+      "content": "<|reserved_special_token_141|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128150": {
+      "content": "<|reserved_special_token_142|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128151": {
+      "content": "<|reserved_special_token_143|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128152": {
+      "content": "<|reserved_special_token_144|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128153": {
+      "content": "<|reserved_special_token_145|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128154": {
+      "content": "<|reserved_special_token_146|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128155": {
+      "content": "<|reserved_special_token_147|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128156": {
+      "content": "<|reserved_special_token_148|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128157": {
+      "content": "<|reserved_special_token_149|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128158": {
+      "content": "<|reserved_special_token_150|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128159": {
+      "content": "<|reserved_special_token_151|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128160": {
+      "content": "<|reserved_special_token_152|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128161": {
+      "content": "<|reserved_special_token_153|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128162": {
+      "content": "<|reserved_special_token_154|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128163": {
+      "content": "<|reserved_special_token_155|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128164": {
+      "content": "<|reserved_special_token_156|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128165": {
+      "content": "<|reserved_special_token_157|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128166": {
+      "content": "<|reserved_special_token_158|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128167": {
+      "content": "<|reserved_special_token_159|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128168": {
+      "content": "<|reserved_special_token_160|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128169": {
+      "content": "<|reserved_special_token_161|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128170": {
+      "content": "<|reserved_special_token_162|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128171": {
+      "content": "<|reserved_special_token_163|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128172": {
+      "content": "<|reserved_special_token_164|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128173": {
+      "content": "<|reserved_special_token_165|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128174": {
+      "content": "<|reserved_special_token_166|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128175": {
+      "content": "<|reserved_special_token_167|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128176": {
+      "content": "<|reserved_special_token_168|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128177": {
+      "content": "<|reserved_special_token_169|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128178": {
+      "content": "<|reserved_special_token_170|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128179": {
+      "content": "<|reserved_special_token_171|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128180": {
+      "content": "<|reserved_special_token_172|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128181": {
+      "content": "<|reserved_special_token_173|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128182": {
+      "content": "<|reserved_special_token_174|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128183": {
+      "content": "<|reserved_special_token_175|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128184": {
+      "content": "<|reserved_special_token_176|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128185": {
+      "content": "<|reserved_special_token_177|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128186": {
+      "content": "<|reserved_special_token_178|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128187": {
+      "content": "<|reserved_special_token_179|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128188": {
+      "content": "<|reserved_special_token_180|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128189": {
+      "content": "<|reserved_special_token_181|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128190": {
+      "content": "<|reserved_special_token_182|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128191": {
+      "content": "<|reserved_special_token_183|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128192": {
+      "content": "<|reserved_special_token_184|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128193": {
+      "content": "<|reserved_special_token_185|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128194": {
+      "content": "<|reserved_special_token_186|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128195": {
+      "content": "<|reserved_special_token_187|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128196": {
+      "content": "<|reserved_special_token_188|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128197": {
+      "content": "<|reserved_special_token_189|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128198": {
+      "content": "<|reserved_special_token_190|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128199": {
+      "content": "<|reserved_special_token_191|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128200": {
+      "content": "<|reserved_special_token_192|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128201": {
+      "content": "<|reserved_special_token_193|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128202": {
+      "content": "<|reserved_special_token_194|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128203": {
+      "content": "<|reserved_special_token_195|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128204": {
+      "content": "<|reserved_special_token_196|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128205": {
+      "content": "<|reserved_special_token_197|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128206": {
+      "content": "<|reserved_special_token_198|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128207": {
+      "content": "<|reserved_special_token_199|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128208": {
+      "content": "<|reserved_special_token_200|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128209": {
+      "content": "<|reserved_special_token_201|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128210": {
+      "content": "<|reserved_special_token_202|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128211": {
+      "content": "<|reserved_special_token_203|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128212": {
+      "content": "<|reserved_special_token_204|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128213": {
+      "content": "<|reserved_special_token_205|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128214": {
+      "content": "<|reserved_special_token_206|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128215": {
+      "content": "<|reserved_special_token_207|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128216": {
+      "content": "<|reserved_special_token_208|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128217": {
+      "content": "<|reserved_special_token_209|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128218": {
+      "content": "<|reserved_special_token_210|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128219": {
+      "content": "<|reserved_special_token_211|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128220": {
+      "content": "<|reserved_special_token_212|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128221": {
+      "content": "<|reserved_special_token_213|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128222": {
+      "content": "<|reserved_special_token_214|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128223": {
+      "content": "<|reserved_special_token_215|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128224": {
+      "content": "<|reserved_special_token_216|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128225": {
+      "content": "<|reserved_special_token_217|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128226": {
+      "content": "<|reserved_special_token_218|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128227": {
+      "content": "<|reserved_special_token_219|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128228": {
+      "content": "<|reserved_special_token_220|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128229": {
+      "content": "<|reserved_special_token_221|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128230": {
+      "content": "<|reserved_special_token_222|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128231": {
+      "content": "<|reserved_special_token_223|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128232": {
+      "content": "<|reserved_special_token_224|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128233": {
+      "content": "<|reserved_special_token_225|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128234": {
+      "content": "<|reserved_special_token_226|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128235": {
+      "content": "<|reserved_special_token_227|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128236": {
+      "content": "<|reserved_special_token_228|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128237": {
+      "content": "<|reserved_special_token_229|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128238": {
+      "content": "<|reserved_special_token_230|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128239": {
+      "content": "<|reserved_special_token_231|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128240": {
+      "content": "<|reserved_special_token_232|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128241": {
+      "content": "<|reserved_special_token_233|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128242": {
+      "content": "<|reserved_special_token_234|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128243": {
+      "content": "<|reserved_special_token_235|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128244": {
+      "content": "<|reserved_special_token_236|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128245": {
+      "content": "<|reserved_special_token_237|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128246": {
+      "content": "<|reserved_special_token_238|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128247": {
+      "content": "<|reserved_special_token_239|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128248": {
+      "content": "<|reserved_special_token_240|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128249": {
+      "content": "<|reserved_special_token_241|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128250": {
+      "content": "<|reserved_special_token_242|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128251": {
+      "content": "<|reserved_special_token_243|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128252": {
+      "content": "<|reserved_special_token_244|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128253": {
+      "content": "<|reserved_special_token_245|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128254": {
+      "content": "<|reserved_special_token_246|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128255": {
+      "content": "<|reserved_special_token_247|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|begin_of_text|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|eot_id|>",
+  "extra_special_tokens": {},
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "pad_token": "<|eot_id|>",
+  "tokenizer_class": "PreTrainedTokenizerFast"
+}

checkpoint-3670/trainer_state.json

@@ -0,0 +1,3346 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 3670,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0002725306171115161,
+      "grad_norm": 3.984375,
+      "learning_rate": 0.0,
+      "loss": 1.0007,
+      "mean_token_accuracy": 0.7572115659713745,
+      "num_tokens": 180497.0,
+      "step": 1
+    },
+    {
+      "epoch": 0.0027253061711151614,
+      "grad_norm": 3.4375,
+      "learning_rate": 2.0361990950226245e-06,
+      "loss": 1.0301,
+      "mean_token_accuracy": 0.7510163293530544,
+      "num_tokens": 1772191.0,
+      "step": 10
+    },
+    {
+      "epoch": 0.005450612342230323,
+      "grad_norm": 2.484375,
+      "learning_rate": 4.298642533936651e-06,
+      "loss": 1.0282,
+      "mean_token_accuracy": 0.7490902768447996,
+      "num_tokens": 3528128.0,
+      "step": 20
+    },
+    {
+      "epoch": 0.008175918513345483,
+      "grad_norm": 1.2734375,
+      "learning_rate": 6.5610859728506795e-06,
+      "loss": 0.9845,
+      "mean_token_accuracy": 0.7558933278545737,
+      "num_tokens": 5321820.0,
+      "step": 30
+    },
+    {
+      "epoch": 0.010901224684460645,
+      "grad_norm": 0.9453125,
+      "learning_rate": 8.823529411764707e-06,
+      "loss": 0.9753,
+      "mean_token_accuracy": 0.7576656749472022,
+      "num_tokens": 7036353.0,
+      "step": 40
+    },
+    {
+      "epoch": 0.013626530855575806,
+      "grad_norm": 0.8671875,
+      "learning_rate": 1.1085972850678733e-05,
+      "loss": 0.9656,
+      "mean_token_accuracy": 0.7573941433802247,
+      "num_tokens": 8794502.0,
+      "step": 50
+    },
+    {
+      "epoch": 0.016351837026690966,
+      "grad_norm": 1.0078125,
+      "learning_rate": 1.3348416289592761e-05,
+      "loss": 0.9552,
+      "mean_token_accuracy": 0.76101697627455,
+      "num_tokens": 10519144.0,
+      "step": 60
+    },
+    {
+      "epoch": 0.019077143197806127,
+      "grad_norm": 0.92578125,
+      "learning_rate": 1.5610859728506788e-05,
+      "loss": 0.9285,
+      "mean_token_accuracy": 0.7672853864729404,
+      "num_tokens": 12272638.0,
+      "step": 70
+    },
+    {
+      "epoch": 0.02180244936892129,
+      "grad_norm": 0.85546875,
+      "learning_rate": 1.7873303167420814e-05,
+      "loss": 0.9288,
+      "mean_token_accuracy": 0.7670242975465953,
+      "num_tokens": 14037595.0,
+      "step": 80
+    },
+    {
+      "epoch": 0.02452775554003645,
+      "grad_norm": 0.9375,
+      "learning_rate": 2.0135746606334844e-05,
+      "loss": 0.9212,
+      "mean_token_accuracy": 0.7678890896961093,
+      "num_tokens": 15813413.0,
+      "step": 90
+    },
+    {
+      "epoch": 0.027253061711151612,
+      "grad_norm": 0.94921875,
+      "learning_rate": 2.239819004524887e-05,
+      "loss": 0.9213,
+      "mean_token_accuracy": 0.7675107896327973,
+      "num_tokens": 17606667.0,
+      "step": 100
+    },
+    {
+      "epoch": 0.029978367882266772,
+      "grad_norm": 0.91015625,
+      "learning_rate": 2.4660633484162897e-05,
+      "loss": 0.8878,
+      "mean_token_accuracy": 0.7746313972398639,
+      "num_tokens": 19387650.0,
+      "step": 110
+    },
+    {
+      "epoch": 0.03270367405338193,
+      "grad_norm": 0.859375,
+      "learning_rate": 2.6923076923076923e-05,
+      "loss": 0.893,
+      "mean_token_accuracy": 0.7730802249163389,
+      "num_tokens": 21111629.0,
+      "step": 120
+    },
+    {
+      "epoch": 0.03542898022449709,
+      "grad_norm": 0.83203125,
+      "learning_rate": 2.9185520361990953e-05,
+      "loss": 0.8808,
+      "mean_token_accuracy": 0.7762882491573692,
+      "num_tokens": 22855847.0,
+      "step": 130
+    },
+    {
+      "epoch": 0.038154286395612254,
+      "grad_norm": 0.875,
+      "learning_rate": 3.1447963800904976e-05,
+      "loss": 0.8721,
+      "mean_token_accuracy": 0.7789254249073565,
+      "num_tokens": 24647294.0,
+      "step": 140
+    },
+    {
+      "epoch": 0.04087959256672742,
+      "grad_norm": 1.0859375,
+      "learning_rate": 3.371040723981901e-05,
+      "loss": 0.8569,
+      "mean_token_accuracy": 0.7798295577988028,
+      "num_tokens": 26336128.0,
+      "step": 150
+    },
+    {
+      "epoch": 0.04360489873784258,
+      "grad_norm": 0.85546875,
+      "learning_rate": 3.5972850678733036e-05,
+      "loss": 0.9039,
+      "mean_token_accuracy": 0.7717110466212034,
+      "num_tokens": 28061528.0,
+      "step": 160
+    },
+    {
+      "epoch": 0.04633020490895774,
+      "grad_norm": 0.8515625,
+      "learning_rate": 3.8235294117647055e-05,
+      "loss": 0.8949,
+      "mean_token_accuracy": 0.7732486605644227,
+      "num_tokens": 29834651.0,
+      "step": 170
+    },
+    {
+      "epoch": 0.0490555110800729,
+      "grad_norm": 0.890625,
+      "learning_rate": 4.049773755656109e-05,
+      "loss": 0.9139,
+      "mean_token_accuracy": 0.7696884664706886,
+      "num_tokens": 31533806.0,
+      "step": 180
+    },
+    {
+      "epoch": 0.05178081725118806,
+      "grad_norm": 0.8984375,
+      "learning_rate": 4.2760180995475115e-05,
+      "loss": 0.8715,
+      "mean_token_accuracy": 0.7762662811204791,
+      "num_tokens": 33328644.0,
+      "step": 190
+    },
+    {
+      "epoch": 0.054506123422303224,
+      "grad_norm": 0.97265625,
+      "learning_rate": 4.502262443438914e-05,
+      "loss": 0.8792,
+      "mean_token_accuracy": 0.7759487941861153,
+      "num_tokens": 35085202.0,
+      "step": 200
+    },
+    {
+      "epoch": 0.057231429593418384,
+      "grad_norm": 0.91796875,
+      "learning_rate": 4.728506787330317e-05,
+      "loss": 0.8983,
+      "mean_token_accuracy": 0.7737127006053924,
+      "num_tokens": 36899685.0,
+      "step": 210
+    },
+    {
+      "epoch": 0.059956735764533545,
+      "grad_norm": 0.84375,
+      "learning_rate": 4.95475113122172e-05,
+      "loss": 0.8259,
+      "mean_token_accuracy": 0.7865078922361135,
+      "num_tokens": 38663867.0,
+      "step": 220
+    },
+    {
+      "epoch": 0.0626820419356487,
+      "grad_norm": 0.91796875,
+      "learning_rate": 4.994381233319287e-05,
+      "loss": 0.847,
+      "mean_token_accuracy": 0.7814395157620311,
+      "num_tokens": 40368688.0,
+      "step": 230
+    },
+    {
+      "epoch": 0.06540734810676387,
+      "grad_norm": 0.90625,
+      "learning_rate": 4.9873577749683945e-05,
+      "loss": 0.8656,
+      "mean_token_accuracy": 0.777296563051641,
+      "num_tokens": 42091556.0,
+      "step": 240
+    },
+    {
+      "epoch": 0.06813265427787903,
+      "grad_norm": 0.8046875,
+      "learning_rate": 4.9803343166175026e-05,
+      "loss": 0.8618,
+      "mean_token_accuracy": 0.7788917429745197,
+      "num_tokens": 43832865.0,
+      "step": 250
+    },
+    {
+      "epoch": 0.07085796044899419,
+      "grad_norm": 0.89453125,
+      "learning_rate": 4.9733108582666106e-05,
+      "loss": 0.8709,
+      "mean_token_accuracy": 0.7771488446742296,
+      "num_tokens": 45556421.0,
+      "step": 260
+    },
+    {
+      "epoch": 0.07358326662010935,
+      "grad_norm": 0.9140625,
+      "learning_rate": 4.9662873999157186e-05,
+      "loss": 0.899,
+      "mean_token_accuracy": 0.7727748015895486,
+      "num_tokens": 47284838.0,
+      "step": 270
+    },
+    {
+      "epoch": 0.07630857279122451,
+      "grad_norm": 0.98828125,
+      "learning_rate": 4.9592639415648266e-05,
+      "loss": 0.8636,
+      "mean_token_accuracy": 0.777201015688479,
+      "num_tokens": 49025592.0,
+      "step": 280
+    },
+    {
+      "epoch": 0.07903387896233968,
+      "grad_norm": 0.99609375,
+      "learning_rate": 4.9522404832139346e-05,
+      "loss": 0.9206,
+      "mean_token_accuracy": 0.7665060743689537,
+      "num_tokens": 50771353.0,
+      "step": 290
+    },
+    {
+      "epoch": 0.08175918513345484,
+      "grad_norm": 0.90234375,
+      "learning_rate": 4.945217024863043e-05,
+      "loss": 0.8601,
+      "mean_token_accuracy": 0.7790314011275769,
+      "num_tokens": 52521467.0,
+      "step": 300
+    },
+    {
+      "epoch": 0.08448449130457,
+      "grad_norm": 0.83984375,
+      "learning_rate": 4.938193566512151e-05,
+      "loss": 0.8739,
+      "mean_token_accuracy": 0.7764257783070206,
+      "num_tokens": 54338858.0,
+      "step": 310
+    },
+    {
+      "epoch": 0.08720979747568516,
+      "grad_norm": 1.015625,
+      "learning_rate": 4.931170108161259e-05,
+      "loss": 0.8778,
+      "mean_token_accuracy": 0.7759509992785751,
+      "num_tokens": 56014081.0,
+      "step": 320
+    },
+    {
+      "epoch": 0.08993510364680032,
+      "grad_norm": 0.98046875,
+      "learning_rate": 4.924146649810367e-05,
+      "loss": 0.8632,
+      "mean_token_accuracy": 0.7777853138744831,
+      "num_tokens": 57778337.0,
+      "step": 330
+    },
+    {
+      "epoch": 0.09266040981791548,
+      "grad_norm": 0.8125,
+      "learning_rate": 4.9171231914594754e-05,
+      "loss": 0.8577,
+      "mean_token_accuracy": 0.7796265617012977,
+      "num_tokens": 59528977.0,
+      "step": 340
+    },
+    {
+      "epoch": 0.09538571598903064,
+      "grad_norm": 0.8203125,
+      "learning_rate": 4.910099733108583e-05,
+      "loss": 0.8736,
+      "mean_token_accuracy": 0.7766836697235704,
+      "num_tokens": 61295985.0,
+      "step": 350
+    },
+    {
+      "epoch": 0.0981110221601458,
+      "grad_norm": 0.91015625,
+      "learning_rate": 4.903076274757691e-05,
+      "loss": 0.8591,
+      "mean_token_accuracy": 0.7787051644176245,
+      "num_tokens": 63026897.0,
+      "step": 360
+    },
+    {
+      "epoch": 0.10083632833126097,
+      "grad_norm": 0.86328125,
+      "learning_rate": 4.896052816406799e-05,
+      "loss": 0.8586,
+      "mean_token_accuracy": 0.7798169434070588,
+      "num_tokens": 64770711.0,
+      "step": 370
+    },
+    {
+      "epoch": 0.10356163450237613,
+      "grad_norm": 0.94921875,
+      "learning_rate": 4.889029358055907e-05,
+      "loss": 0.8552,
+      "mean_token_accuracy": 0.7811101474799216,
+      "num_tokens": 66551008.0,
+      "step": 380
+    },
+    {
+      "epoch": 0.10628694067349129,
+      "grad_norm": 0.85546875,
+      "learning_rate": 4.882005899705015e-05,
+      "loss": 0.8713,
+      "mean_token_accuracy": 0.7761319866403937,
+      "num_tokens": 68320690.0,
+      "step": 390
+    },
+    {
+      "epoch": 0.10901224684460645,
+      "grad_norm": 0.828125,
+      "learning_rate": 4.874982441354123e-05,
+      "loss": 0.8404,
+      "mean_token_accuracy": 0.783243116363883,
+      "num_tokens": 70095284.0,
+      "step": 400
+    },
+    {
+      "epoch": 0.11173755301572161,
+      "grad_norm": 0.8515625,
+      "learning_rate": 4.8679589830032316e-05,
+      "loss": 0.8557,
+      "mean_token_accuracy": 0.7806508645415307,
+      "num_tokens": 71803511.0,
+      "step": 410
+    },
+    {
+      "epoch": 0.11446285918683677,
+      "grad_norm": 0.85546875,
+      "learning_rate": 4.860935524652339e-05,
+      "loss": 0.8243,
+      "mean_token_accuracy": 0.7869562221691012,
+      "num_tokens": 73528297.0,
+      "step": 420
+    },
+    {
+      "epoch": 0.11718816535795193,
+      "grad_norm": 0.83984375,
+      "learning_rate": 4.853912066301447e-05,
+      "loss": 0.8256,
+      "mean_token_accuracy": 0.7865307861939073,
+      "num_tokens": 75274289.0,
+      "step": 430
+    },
+    {
+      "epoch": 0.11991347152906709,
+      "grad_norm": 0.921875,
+      "learning_rate": 4.846888607950555e-05,
+      "loss": 0.8632,
+      "mean_token_accuracy": 0.7779594603925943,
+      "num_tokens": 77077403.0,
+      "step": 440
+    },
+    {
+      "epoch": 0.12263877770018225,
+      "grad_norm": 0.89453125,
+      "learning_rate": 4.8398651495996636e-05,
+      "loss": 0.8605,
+      "mean_token_accuracy": 0.7778984166681766,
+      "num_tokens": 78859882.0,
+      "step": 450
+    },
+    {
+      "epoch": 0.1253640838712974,
+      "grad_norm": 0.87890625,
+      "learning_rate": 4.832841691248771e-05,
+      "loss": 0.8768,
+      "mean_token_accuracy": 0.7754263358190656,
+      "num_tokens": 80590976.0,
+      "step": 460
+    },
+    {
+      "epoch": 0.12808939004241257,
+      "grad_norm": 0.984375,
+      "learning_rate": 4.825818232897879e-05,
+      "loss": 0.8368,
+      "mean_token_accuracy": 0.7832479283213616,
+      "num_tokens": 82275126.0,
+      "step": 470
+    },
+    {
+      "epoch": 0.13081469621352773,
+      "grad_norm": 0.88671875,
+      "learning_rate": 4.818794774546987e-05,
+      "loss": 0.8395,
+      "mean_token_accuracy": 0.7824733754619956,
+      "num_tokens": 84096910.0,
+      "step": 480
+    },
+    {
+      "epoch": 0.1335400023846429,
+      "grad_norm": 0.84765625,
+      "learning_rate": 4.811771316196095e-05,
+      "loss": 0.8625,
+      "mean_token_accuracy": 0.7794647924602032,
+      "num_tokens": 85876326.0,
+      "step": 490
+    },
+    {
+      "epoch": 0.13626530855575805,
+      "grad_norm": 0.86328125,
+      "learning_rate": 4.804747857845203e-05,
+      "loss": 0.8712,
+      "mean_token_accuracy": 0.7776954183354974,
+      "num_tokens": 87607478.0,
+      "step": 500
+    },
+    {
+      "epoch": 0.1389906147268732,
+      "grad_norm": 0.83984375,
+      "learning_rate": 4.797724399494311e-05,
+      "loss": 0.84,
+      "mean_token_accuracy": 0.7830386471003294,
+      "num_tokens": 89350066.0,
+      "step": 510
+    },
+    {
+      "epoch": 0.14171592089798837,
+      "grad_norm": 0.89453125,
+      "learning_rate": 4.79070094114342e-05,
+      "loss": 0.8511,
+      "mean_token_accuracy": 0.7820526211522519,
+      "num_tokens": 91067910.0,
+      "step": 520
+    },
+    {
+      "epoch": 0.14444122706910353,
+      "grad_norm": 0.8359375,
+      "learning_rate": 4.783677482792527e-05,
+      "loss": 0.8435,
+      "mean_token_accuracy": 0.7820645566098392,
+      "num_tokens": 92797017.0,
+      "step": 530
+    },
+    {
+      "epoch": 0.1471665332402187,
+      "grad_norm": 0.8359375,
+      "learning_rate": 4.776654024441635e-05,
+      "loss": 0.8569,
+      "mean_token_accuracy": 0.7780541488900781,
+      "num_tokens": 94606329.0,
+      "step": 540
+    },
+    {
+      "epoch": 0.14989183941133385,
+      "grad_norm": 0.8515625,
+      "learning_rate": 4.769630566090743e-05,
+      "loss": 0.8405,
+      "mean_token_accuracy": 0.7827091997489333,
+      "num_tokens": 96331087.0,
+      "step": 550
+    },
+    {
+      "epoch": 0.15261714558244902,
+      "grad_norm": 0.86328125,
+      "learning_rate": 4.762607107739852e-05,
+      "loss": 0.8563,
+      "mean_token_accuracy": 0.7789288500323892,
+      "num_tokens": 98138711.0,
+      "step": 560
+    },
+    {
+      "epoch": 0.15534245175356418,
+      "grad_norm": 0.83984375,
+      "learning_rate": 4.755583649388959e-05,
+      "loss": 0.8597,
+      "mean_token_accuracy": 0.7786139996722341,
+      "num_tokens": 99885005.0,
+      "step": 570
+    },
+    {
+      "epoch": 0.15806775792467936,
+      "grad_norm": 0.8125,
+      "learning_rate": 4.748560191038067e-05,
+      "loss": 0.7849,
+      "mean_token_accuracy": 0.794182694517076,
+      "num_tokens": 101636075.0,
+      "step": 580
+    },
+    {
+      "epoch": 0.16079306409579452,
+      "grad_norm": 0.96875,
+      "learning_rate": 4.741536732687175e-05,
+      "loss": 0.837,
+      "mean_token_accuracy": 0.7835318537428975,
+      "num_tokens": 103349118.0,
+      "step": 590
+    },
+    {
+      "epoch": 0.16351837026690969,
+      "grad_norm": 0.90625,
+      "learning_rate": 4.734513274336283e-05,
+      "loss": 0.8477,
+      "mean_token_accuracy": 0.7813424309715629,
+      "num_tokens": 105033010.0,
+      "step": 600
+    },
+    {
+      "epoch": 0.16624367643802485,
+      "grad_norm": 0.86328125,
+      "learning_rate": 4.727489815985391e-05,
+      "loss": 0.817,
+      "mean_token_accuracy": 0.7874910736456513,
+      "num_tokens": 106723283.0,
+      "step": 610
+    },
+    {
+      "epoch": 0.16896898260914,
+      "grad_norm": 0.83984375,
+      "learning_rate": 4.720466357634499e-05,
+      "loss": 0.8385,
+      "mean_token_accuracy": 0.782614064309746,
+      "num_tokens": 108436878.0,
+      "step": 620
+    },
+    {
+      "epoch": 0.17169428878025517,
+      "grad_norm": 0.875,
+      "learning_rate": 4.713442899283608e-05,
+      "loss": 0.8223,
+      "mean_token_accuracy": 0.7862238166853786,
+      "num_tokens": 110203157.0,
+      "step": 630
+    },
+    {
+      "epoch": 0.17441959495137033,
+      "grad_norm": 1.015625,
+      "learning_rate": 4.706419440932715e-05,
+      "loss": 0.8492,
+      "mean_token_accuracy": 0.781964666210115,
+      "num_tokens": 111949130.0,
+      "step": 640
+    },
+    {
+      "epoch": 0.1771449011224855,
+      "grad_norm": 0.8671875,
+      "learning_rate": 4.6993959825818233e-05,
+      "loss": 0.8455,
+      "mean_token_accuracy": 0.7806475246325135,
+      "num_tokens": 113652926.0,
+      "step": 650
+    },
+    {
+      "epoch": 0.17987020729360065,
+      "grad_norm": 0.859375,
+      "learning_rate": 4.6923725242309314e-05,
+      "loss": 0.8362,
+      "mean_token_accuracy": 0.7815606592223048,
+      "num_tokens": 115334647.0,
+      "step": 660
+    },
+    {
+      "epoch": 0.1825955134647158,
+      "grad_norm": 0.82421875,
+      "learning_rate": 4.68534906588004e-05,
+      "loss": 0.8046,
+      "mean_token_accuracy": 0.7903961345553399,
+      "num_tokens": 117032381.0,
+      "step": 670
+    },
+    {
+      "epoch": 0.18532081963583097,
+      "grad_norm": 0.828125,
+      "learning_rate": 4.6783256075291474e-05,
+      "loss": 0.8341,
+      "mean_token_accuracy": 0.7830562081187964,
+      "num_tokens": 118801553.0,
+      "step": 680
+    },
+    {
+      "epoch": 0.18804612580694613,
+      "grad_norm": 0.9140625,
+      "learning_rate": 4.6713021491782554e-05,
+      "loss": 0.8481,
+      "mean_token_accuracy": 0.7801631511189043,
+      "num_tokens": 120530271.0,
+      "step": 690
+    },
+    {
+      "epoch": 0.1907714319780613,
+      "grad_norm": 0.78515625,
+      "learning_rate": 4.6642786908273634e-05,
+      "loss": 0.8384,
+      "mean_token_accuracy": 0.78369100689888,
+      "num_tokens": 122314411.0,
+      "step": 700
+    },
+    {
+      "epoch": 0.19349673814917645,
+      "grad_norm": 0.81640625,
+      "learning_rate": 4.6572552324764715e-05,
+      "loss": 0.8339,
+      "mean_token_accuracy": 0.7835866080597043,
+      "num_tokens": 124054113.0,
+      "step": 710
+    },
+    {
+      "epoch": 0.1962220443202916,
+      "grad_norm": 0.8984375,
+      "learning_rate": 4.6502317741255795e-05,
+      "loss": 0.8506,
+      "mean_token_accuracy": 0.7801098952069878,
+      "num_tokens": 125786705.0,
+      "step": 720
+    },
+    {
+      "epoch": 0.19894735049140677,
+      "grad_norm": 0.84375,
+      "learning_rate": 4.6432083157746875e-05,
+      "loss": 0.8487,
+      "mean_token_accuracy": 0.779451628960669,
+      "num_tokens": 127510112.0,
+      "step": 730
+    },
+    {
+      "epoch": 0.20167265666252193,
+      "grad_norm": 0.859375,
+      "learning_rate": 4.636184857423796e-05,
+      "loss": 0.8196,
+      "mean_token_accuracy": 0.7867877034470439,
+      "num_tokens": 129321733.0,
+      "step": 740
+    },
+    {
+      "epoch": 0.2043979628336371,
+      "grad_norm": 0.7890625,
+      "learning_rate": 4.6291613990729035e-05,
+      "loss": 0.852,
+      "mean_token_accuracy": 0.7805778596550226,
+      "num_tokens": 131068939.0,
+      "step": 750
+    },
+    {
+      "epoch": 0.20712326900475225,
+      "grad_norm": 0.90234375,
+      "learning_rate": 4.6221379407220116e-05,
+      "loss": 0.8814,
+      "mean_token_accuracy": 0.774320712685585,
+      "num_tokens": 132800192.0,
+      "step": 760
+    },
+    {
+      "epoch": 0.2098485751758674,
+      "grad_norm": 0.87109375,
+      "learning_rate": 4.6151144823711196e-05,
+      "loss": 0.8267,
+      "mean_token_accuracy": 0.785618613474071,
+      "num_tokens": 134501880.0,
+      "step": 770
+    },
+    {
+      "epoch": 0.21257388134698257,
+      "grad_norm": 0.859375,
+      "learning_rate": 4.608091024020228e-05,
+      "loss": 0.8074,
+      "mean_token_accuracy": 0.7888548966497183,
+      "num_tokens": 136227230.0,
+      "step": 780
+    },
+    {
+      "epoch": 0.21529918751809773,
+      "grad_norm": 0.90625,
+      "learning_rate": 4.6010675656693356e-05,
+      "loss": 0.8212,
+      "mean_token_accuracy": 0.7867167362943291,
+      "num_tokens": 137938433.0,
+      "step": 790
+    },
+    {
+      "epoch": 0.2180244936892129,
+      "grad_norm": 0.83203125,
+      "learning_rate": 4.5940441073184436e-05,
+      "loss": 0.8301,
+      "mean_token_accuracy": 0.7851111644878983,
+      "num_tokens": 139667163.0,
+      "step": 800
+    },
+    {
+      "epoch": 0.22074979986032806,
+      "grad_norm": 0.74609375,
+      "learning_rate": 4.5870206489675517e-05,
+      "loss": 0.8087,
+      "mean_token_accuracy": 0.7898392202332616,
+      "num_tokens": 141481099.0,
+      "step": 810
+    },
+    {
+      "epoch": 0.22347510603144322,
+      "grad_norm": 0.8515625,
+      "learning_rate": 4.57999719061666e-05,
+      "loss": 0.863,
+      "mean_token_accuracy": 0.7779477247968316,
+      "num_tokens": 143204243.0,
+      "step": 820
+    },
+    {
+      "epoch": 0.22620041220255838,
+      "grad_norm": 0.84375,
+      "learning_rate": 4.572973732265768e-05,
+      "loss": 0.83,
+      "mean_token_accuracy": 0.7857123276218771,
+      "num_tokens": 144995581.0,
+      "step": 830
+    },
+    {
+      "epoch": 0.22892571837367354,
+      "grad_norm": 0.859375,
+      "learning_rate": 4.565950273914876e-05,
+      "loss": 0.8342,
+      "mean_token_accuracy": 0.7827581511810422,
+      "num_tokens": 146711076.0,
+      "step": 840
+    },
+    {
+      "epoch": 0.2316510245447887,
+      "grad_norm": 0.7734375,
+      "learning_rate": 4.5589268155639844e-05,
+      "loss": 0.8351,
+      "mean_token_accuracy": 0.7841356256976724,
+      "num_tokens": 148463902.0,
+      "step": 850
+    },
+    {
+      "epoch": 0.23437633071590386,
+      "grad_norm": 0.86328125,
+      "learning_rate": 4.551903357213092e-05,
+      "loss": 0.8552,
+      "mean_token_accuracy": 0.7791399486362934,
+      "num_tokens": 150212801.0,
+      "step": 860
+    },
+    {
+      "epoch": 0.23710163688701902,
+      "grad_norm": 0.79296875,
+      "learning_rate": 4.5448798988622e-05,
+      "loss": 0.808,
+      "mean_token_accuracy": 0.7881301037967205,
+      "num_tokens": 151950016.0,
+      "step": 870
+    },
+    {
+      "epoch": 0.23982694305813418,
+      "grad_norm": 0.90625,
+      "learning_rate": 4.537856440511308e-05,
+      "loss": 0.8184,
+      "mean_token_accuracy": 0.786030488088727,
+      "num_tokens": 153686341.0,
+      "step": 880
+    },
+    {
+      "epoch": 0.24255224922924934,
+      "grad_norm": 0.7578125,
+      "learning_rate": 4.5308329821604165e-05,
+      "loss": 0.8432,
+      "mean_token_accuracy": 0.7818001460283994,
+      "num_tokens": 155442220.0,
+      "step": 890
+    },
+    {
+      "epoch": 0.2452775554003645,
+      "grad_norm": 0.87890625,
+      "learning_rate": 4.523809523809524e-05,
+      "loss": 0.8182,
+      "mean_token_accuracy": 0.786651942692697,
+      "num_tokens": 157222278.0,
+      "step": 900
+    },
+    {
+      "epoch": 0.24800286157147966,
+      "grad_norm": 0.81640625,
+      "learning_rate": 4.516786065458632e-05,
+      "loss": 0.8115,
+      "mean_token_accuracy": 0.787405708245933,
+      "num_tokens": 158989873.0,
+      "step": 910
+    },
+    {
+      "epoch": 0.2507281677425948,
+      "grad_norm": 1.0,
+      "learning_rate": 4.50976260710774e-05,
+      "loss": 0.8385,
+      "mean_token_accuracy": 0.7838497390039265,
+      "num_tokens": 160740690.0,
+      "step": 920
+    },
+    {
+      "epoch": 0.25345347391371,
+      "grad_norm": 0.8515625,
+      "learning_rate": 4.502739148756848e-05,
+      "loss": 0.8375,
+      "mean_token_accuracy": 0.782560715265572,
+      "num_tokens": 162505210.0,
+      "step": 930
+    },
+    {
+      "epoch": 0.25617878008482514,
+      "grad_norm": 0.87890625,
+      "learning_rate": 4.495715690405956e-05,
+      "loss": 0.8488,
+      "mean_token_accuracy": 0.7806899799033999,
+      "num_tokens": 164184119.0,
+      "step": 940
+    },
+    {
+      "epoch": 0.2589040862559403,
+      "grad_norm": 0.80078125,
+      "learning_rate": 4.488692232055064e-05,
+      "loss": 0.8067,
+      "mean_token_accuracy": 0.7890315353870392,
+      "num_tokens": 165971146.0,
+      "step": 950
+    },
+    {
+      "epoch": 0.26162939242705546,
+      "grad_norm": 0.875,
+      "learning_rate": 4.4816687737041726e-05,
+      "loss": 0.84,
+      "mean_token_accuracy": 0.7840523752383888,
+      "num_tokens": 167701701.0,
+      "step": 960
+    },
+    {
+      "epoch": 0.2643546985981706,
+      "grad_norm": 2.984375,
+      "learning_rate": 4.47464531535328e-05,
+      "loss": 0.8313,
+      "mean_token_accuracy": 0.7840032324194908,
+      "num_tokens": 169443326.0,
+      "step": 970
+    },
+    {
+      "epoch": 0.2670800047692858,
+      "grad_norm": 0.828125,
+      "learning_rate": 4.467621857002388e-05,
+      "loss": 0.8314,
+      "mean_token_accuracy": 0.7835690898820757,
+      "num_tokens": 171199385.0,
+      "step": 980
+    },
+    {
+      "epoch": 0.26980531094040094,
+      "grad_norm": 0.86328125,
+      "learning_rate": 4.460598398651496e-05,
+      "loss": 0.8055,
+      "mean_token_accuracy": 0.7899746172130108,
+      "num_tokens": 172929493.0,
+      "step": 990
+    },
+    {
+      "epoch": 0.2725306171115161,
+      "grad_norm": 0.87109375,
+      "learning_rate": 4.453574940300605e-05,
+      "loss": 0.8115,
+      "mean_token_accuracy": 0.7882102673873306,
+      "num_tokens": 174672601.0,
+      "step": 1000
+    },
+    {
+      "epoch": 0.27525592328263127,
+      "grad_norm": 0.7734375,
+      "learning_rate": 4.446551481949712e-05,
+      "loss": 0.8196,
+      "mean_token_accuracy": 0.7871743559837341,
+      "num_tokens": 176441962.0,
+      "step": 1010
+    },
+    {
+      "epoch": 0.2779812294537464,
+      "grad_norm": 0.8984375,
+      "learning_rate": 4.43952802359882e-05,
+      "loss": 0.8143,
+      "mean_token_accuracy": 0.787607959471643,
+      "num_tokens": 178197361.0,
+      "step": 1020
+    },
+    {
+      "epoch": 0.2807065356248616,
+      "grad_norm": 0.92578125,
+      "learning_rate": 4.432504565247929e-05,
+      "loss": 0.8414,
+      "mean_token_accuracy": 0.7807860219851136,
+      "num_tokens": 179887580.0,
+      "step": 1030
+    },
+    {
+      "epoch": 0.28343184179597675,
+      "grad_norm": 0.85546875,
+      "learning_rate": 4.425481106897036e-05,
+      "loss": 0.807,
+      "mean_token_accuracy": 0.7890161452814937,
+      "num_tokens": 181655865.0,
+      "step": 1040
+    },
+    {
+      "epoch": 0.2861571479670919,
+      "grad_norm": 0.9609375,
+      "learning_rate": 4.418457648546144e-05,
+      "loss": 0.8264,
+      "mean_token_accuracy": 0.7865914016962051,
+      "num_tokens": 183445880.0,
+      "step": 1050
+    },
+    {
+      "epoch": 0.28888245413820707,
+      "grad_norm": 0.83203125,
+      "learning_rate": 4.411434190195252e-05,
+      "loss": 0.7997,
+      "mean_token_accuracy": 0.7905214043334127,
+      "num_tokens": 185175706.0,
+      "step": 1060
+    },
+    {
+      "epoch": 0.29160776030932223,
+      "grad_norm": 0.87109375,
+      "learning_rate": 4.404410731844361e-05,
+      "loss": 0.8108,
+      "mean_token_accuracy": 0.7874145632609725,
+      "num_tokens": 186873396.0,
+      "step": 1070
+    },
+    {
+      "epoch": 0.2943330664804374,
+      "grad_norm": 0.83984375,
+      "learning_rate": 4.397387273493468e-05,
+      "loss": 0.8398,
+      "mean_token_accuracy": 0.7823239839635789,
+      "num_tokens": 188591288.0,
+      "step": 1080
+    },
+    {
+      "epoch": 0.29705837265155255,
+      "grad_norm": 0.86328125,
+      "learning_rate": 4.390363815142576e-05,
+      "loss": 0.835,
+      "mean_token_accuracy": 0.7830085203051567,
+      "num_tokens": 190375182.0,
+      "step": 1090
+    },
+    {
+      "epoch": 0.2997836788226677,
+      "grad_norm": 0.85546875,
+      "learning_rate": 4.383340356791684e-05,
+      "loss": 0.8234,
+      "mean_token_accuracy": 0.7848609953187407,
+      "num_tokens": 192104609.0,
+      "step": 1100
+    },
+    {
+      "epoch": 0.30250898499378287,
+      "grad_norm": 0.81640625,
+      "learning_rate": 4.376316898440793e-05,
+      "loss": 0.7734,
+      "mean_token_accuracy": 0.7958569876849652,
+      "num_tokens": 193819768.0,
+      "step": 1110
+    },
+    {
+      "epoch": 0.30523429116489803,
+      "grad_norm": 0.83203125,
+      "learning_rate": 4.3692934400899e-05,
+      "loss": 0.8265,
+      "mean_token_accuracy": 0.7864869978278876,
+      "num_tokens": 195541705.0,
+      "step": 1120
+    },
+    {
+      "epoch": 0.3079595973360132,
+      "grad_norm": 0.8125,
+      "learning_rate": 4.362269981739008e-05,
+      "loss": 0.8132,
+      "mean_token_accuracy": 0.787400315888226,
+      "num_tokens": 197250976.0,
+      "step": 1130
+    },
+    {
+      "epoch": 0.31068490350712835,
+      "grad_norm": 0.76171875,
+      "learning_rate": 4.355246523388117e-05,
+      "loss": 0.7897,
+      "mean_token_accuracy": 0.7932030737400055,
+      "num_tokens": 198964087.0,
+      "step": 1140
+    },
+    {
+      "epoch": 0.31341020967824357,
+      "grad_norm": 0.84765625,
+      "learning_rate": 4.348223065037224e-05,
+      "loss": 0.83,
+      "mean_token_accuracy": 0.7837063876911998,
+      "num_tokens": 200654341.0,
+      "step": 1150
+    },
+    {
+      "epoch": 0.31613551584935873,
+      "grad_norm": 0.84765625,
+      "learning_rate": 4.3411996066863323e-05,
+      "loss": 0.8526,
+      "mean_token_accuracy": 0.7796476520597935,
+      "num_tokens": 202410389.0,
+      "step": 1160
+    },
+    {
+      "epoch": 0.3188608220204739,
+      "grad_norm": 0.80078125,
+      "learning_rate": 4.3341761483354404e-05,
+      "loss": 0.8102,
+      "mean_token_accuracy": 0.7881662769243121,
+      "num_tokens": 204176073.0,
+      "step": 1170
+    },
+    {
+      "epoch": 0.32158612819158905,
+      "grad_norm": 0.8515625,
+      "learning_rate": 4.327152689984549e-05,
+      "loss": 0.8067,
+      "mean_token_accuracy": 0.7892425773665309,
+      "num_tokens": 206017214.0,
+      "step": 1180
+    },
+    {
+      "epoch": 0.3243114343627042,
+      "grad_norm": 0.859375,
+      "learning_rate": 4.3201292316336564e-05,
+      "loss": 0.814,
+      "mean_token_accuracy": 0.7871673194691539,
+      "num_tokens": 207832984.0,
+      "step": 1190
+    },
+    {
+      "epoch": 0.32703674053381937,
+      "grad_norm": 0.875,
+      "learning_rate": 4.3131057732827644e-05,
+      "loss": 0.8292,
+      "mean_token_accuracy": 0.7851271273568272,
+      "num_tokens": 209533427.0,
+      "step": 1200
+    },
+    {
+      "epoch": 0.32976204670493453,
+      "grad_norm": 0.8046875,
+      "learning_rate": 4.3060823149318724e-05,
+      "loss": 0.805,
+      "mean_token_accuracy": 0.7889353599399328,
+      "num_tokens": 211344332.0,
+      "step": 1210
+    },
+    {
+      "epoch": 0.3324873528760497,
+      "grad_norm": 0.90234375,
+      "learning_rate": 4.299058856580981e-05,
+      "loss": 0.8338,
+      "mean_token_accuracy": 0.7839144259691239,
+      "num_tokens": 213088746.0,
+      "step": 1220
+    },
+    {
+      "epoch": 0.33521265904716485,
+      "grad_norm": 0.859375,
+      "learning_rate": 4.2920353982300885e-05,
+      "loss": 0.8328,
+      "mean_token_accuracy": 0.7829629480838776,
+      "num_tokens": 214889528.0,
+      "step": 1230
+    },
+    {
+      "epoch": 0.33793796521828,
+      "grad_norm": 0.921875,
+      "learning_rate": 4.2850119398791965e-05,
+      "loss": 0.8395,
+      "mean_token_accuracy": 0.7827292851172387,
+      "num_tokens": 216603081.0,
+      "step": 1240
+    },
+    {
+      "epoch": 0.3406632713893952,
+      "grad_norm": 0.8359375,
+      "learning_rate": 4.277988481528305e-05,
+      "loss": 0.7947,
+      "mean_token_accuracy": 0.7911426387727261,
+      "num_tokens": 218292845.0,
+      "step": 1250
+    },
+    {
+      "epoch": 0.34338857756051033,
+      "grad_norm": 0.87890625,
+      "learning_rate": 4.2709650231774125e-05,
+      "loss": 0.8188,
+      "mean_token_accuracy": 0.7871939340606332,
+      "num_tokens": 220029696.0,
+      "step": 1260
+    },
+    {
+      "epoch": 0.3461138837316255,
+      "grad_norm": 0.80078125,
+      "learning_rate": 4.2639415648265206e-05,
+      "loss": 0.8668,
+      "mean_token_accuracy": 0.777739137224853,
+      "num_tokens": 221810553.0,
+      "step": 1270
+    },
+    {
+      "epoch": 0.34883918990274065,
+      "grad_norm": 0.78515625,
+      "learning_rate": 4.2569181064756286e-05,
+      "loss": 0.8164,
+      "mean_token_accuracy": 0.7896359104663133,
+      "num_tokens": 223552801.0,
+      "step": 1280
+    },
+    {
+      "epoch": 0.3515644960738558,
+      "grad_norm": 0.90234375,
+      "learning_rate": 4.249894648124737e-05,
+      "loss": 0.8235,
+      "mean_token_accuracy": 0.7865501779131592,
+      "num_tokens": 225254929.0,
+      "step": 1290
+    },
+    {
+      "epoch": 0.354289802244971,
+      "grad_norm": 0.86328125,
+      "learning_rate": 4.2428711897738446e-05,
+      "loss": 0.8373,
+      "mean_token_accuracy": 0.78308325111866,
+      "num_tokens": 227017304.0,
+      "step": 1300
+    },
+    {
+      "epoch": 0.35701510841608614,
+      "grad_norm": 0.9296875,
+      "learning_rate": 4.2358477314229526e-05,
+      "loss": 0.7953,
+      "mean_token_accuracy": 0.7921060561202466,
+      "num_tokens": 228785771.0,
+      "step": 1310
+    },
+    {
+      "epoch": 0.3597404145872013,
+      "grad_norm": 0.88671875,
+      "learning_rate": 4.2288242730720607e-05,
+      "loss": 0.8051,
+      "mean_token_accuracy": 0.7898596948012709,
+      "num_tokens": 230489667.0,
+      "step": 1320
+    },
+    {
+      "epoch": 0.36246572075831646,
+      "grad_norm": 0.78125,
+      "learning_rate": 4.2218008147211694e-05,
+      "loss": 0.8204,
+      "mean_token_accuracy": 0.7865555426105857,
+      "num_tokens": 232269581.0,
+      "step": 1330
+    },
+    {
+      "epoch": 0.3651910269294316,
+      "grad_norm": 0.828125,
+      "learning_rate": 4.214777356370277e-05,
+      "loss": 0.8237,
+      "mean_token_accuracy": 0.7836556995287538,
+      "num_tokens": 234028126.0,
+      "step": 1340
+    },
+    {
+      "epoch": 0.3679163331005468,
+      "grad_norm": 0.96484375,
+      "learning_rate": 4.207753898019385e-05,
+      "loss": 0.8403,
+      "mean_token_accuracy": 0.7825251746922731,
+      "num_tokens": 235817129.0,
+      "step": 1350
+    },
+    {
+      "epoch": 0.37064163927166194,
+      "grad_norm": 0.78125,
+      "learning_rate": 4.2007304396684934e-05,
+      "loss": 0.8213,
+      "mean_token_accuracy": 0.786504202708602,
+      "num_tokens": 237588984.0,
+      "step": 1360
+    },
+    {
+      "epoch": 0.3733669454427771,
+      "grad_norm": 0.921875,
+      "learning_rate": 4.193706981317601e-05,
+      "loss": 0.7944,
+      "mean_token_accuracy": 0.7915661975741386,
+      "num_tokens": 239305200.0,
+      "step": 1370
+    },
+    {
+      "epoch": 0.37609225161389226,
+      "grad_norm": 0.79296875,
+      "learning_rate": 4.186683522966709e-05,
+      "loss": 0.812,
+      "mean_token_accuracy": 0.787874061614275,
+      "num_tokens": 241078759.0,
+      "step": 1380
+    },
+    {
+      "epoch": 0.3788175577850074,
+      "grad_norm": 0.859375,
+      "learning_rate": 4.179660064615817e-05,
+      "loss": 0.8313,
+      "mean_token_accuracy": 0.7842249834910036,
+      "num_tokens": 242876841.0,
+      "step": 1390
+    },
+    {
+      "epoch": 0.3815428639561226,
+      "grad_norm": 0.82421875,
+      "learning_rate": 4.1726366062649255e-05,
+      "loss": 0.8042,
+      "mean_token_accuracy": 0.7883623953908682,
+      "num_tokens": 244578724.0,
+      "step": 1400
+    },
+    {
+      "epoch": 0.38426817012723774,
+      "grad_norm": 0.796875,
+      "learning_rate": 4.165613147914033e-05,
+      "loss": 0.8257,
+      "mean_token_accuracy": 0.7838440110906959,
+      "num_tokens": 246363009.0,
+      "step": 1410
+    },
+    {
+      "epoch": 0.3869934762983529,
+      "grad_norm": 0.8125,
+      "learning_rate": 4.158589689563141e-05,
+      "loss": 0.8465,
+      "mean_token_accuracy": 0.7810628946870566,
+      "num_tokens": 248120613.0,
+      "step": 1420
+    },
+    {
+      "epoch": 0.38971878246946806,
+      "grad_norm": 0.82421875,
+      "learning_rate": 4.151566231212249e-05,
+      "loss": 0.8242,
+      "mean_token_accuracy": 0.7861620271578431,
+      "num_tokens": 249941867.0,
+      "step": 1430
+    },
+    {
+      "epoch": 0.3924440886405832,
+      "grad_norm": 0.76953125,
+      "learning_rate": 4.1445427728613576e-05,
+      "loss": 0.7972,
+      "mean_token_accuracy": 0.791112239845097,
+      "num_tokens": 251695734.0,
+      "step": 1440
+    },
+    {
+      "epoch": 0.3951693948116984,
+      "grad_norm": 0.76171875,
+      "learning_rate": 4.137519314510465e-05,
+      "loss": 0.8383,
+      "mean_token_accuracy": 0.7825974946841597,
+      "num_tokens": 253503211.0,
+      "step": 1450
+    },
+    {
+      "epoch": 0.39789470098281354,
+      "grad_norm": 0.95703125,
+      "learning_rate": 4.130495856159573e-05,
+      "loss": 0.8097,
+      "mean_token_accuracy": 0.7883420048281551,
+      "num_tokens": 255313734.0,
+      "step": 1460
+    },
+    {
+      "epoch": 0.4006200071539287,
+      "grad_norm": 0.7578125,
+      "learning_rate": 4.1234723978086816e-05,
+      "loss": 0.7683,
+      "mean_token_accuracy": 0.7976343577727676,
+      "num_tokens": 257016513.0,
+      "step": 1470
+    },
+    {
+      "epoch": 0.40334531332504386,
+      "grad_norm": 0.8984375,
+      "learning_rate": 4.1164489394577896e-05,
+      "loss": 0.836,
+      "mean_token_accuracy": 0.7826737241819501,
+      "num_tokens": 258760362.0,
+      "step": 1480
+    },
+    {
+      "epoch": 0.406070619496159,
+      "grad_norm": 0.765625,
+      "learning_rate": 4.109425481106897e-05,
+      "loss": 0.7991,
+      "mean_token_accuracy": 0.7910492675378918,
+      "num_tokens": 260546971.0,
+      "step": 1490
+    },
+    {
+      "epoch": 0.4087959256672742,
+      "grad_norm": 0.79296875,
+      "learning_rate": 4.102402022756005e-05,
+      "loss": 0.8094,
+      "mean_token_accuracy": 0.7876715689897538,
+      "num_tokens": 262307672.0,
+      "step": 1500
+    },
+    {
+      "epoch": 0.41152123183838935,
+      "grad_norm": 0.84765625,
+      "learning_rate": 4.095378564405114e-05,
+      "loss": 0.8231,
+      "mean_token_accuracy": 0.7858624435961247,
+      "num_tokens": 264023889.0,
+      "step": 1510
+    },
+    {
+      "epoch": 0.4142465380095045,
+      "grad_norm": 0.82421875,
+      "learning_rate": 4.088355106054221e-05,
+      "loss": 0.8192,
+      "mean_token_accuracy": 0.7857081668451429,
+      "num_tokens": 265798519.0,
+      "step": 1520
+    },
+    {
+      "epoch": 0.41697184418061967,
+      "grad_norm": 0.81640625,
+      "learning_rate": 4.081331647703329e-05,
+      "loss": 0.8352,
+      "mean_token_accuracy": 0.7830274883657694,
+      "num_tokens": 267546100.0,
+      "step": 1530
+    },
+    {
+      "epoch": 0.4196971503517348,
+      "grad_norm": 0.78125,
+      "learning_rate": 4.074308189352437e-05,
+      "loss": 0.8015,
+      "mean_token_accuracy": 0.7893059702590108,
+      "num_tokens": 269297641.0,
+      "step": 1540
+    },
+    {
+      "epoch": 0.42242245652285,
+      "grad_norm": 0.734375,
+      "learning_rate": 4.067284731001546e-05,
+      "loss": 0.8352,
+      "mean_token_accuracy": 0.7822487531229854,
+      "num_tokens": 271053006.0,
+      "step": 1550
+    },
+    {
+      "epoch": 0.42514776269396515,
+      "grad_norm": 0.89453125,
+      "learning_rate": 4.060261272650653e-05,
+      "loss": 0.8188,
+      "mean_token_accuracy": 0.7875180309638381,
+      "num_tokens": 272829487.0,
+      "step": 1560
+    },
+    {
+      "epoch": 0.4278730688650803,
+      "grad_norm": 0.82421875,
+      "learning_rate": 4.053237814299761e-05,
+      "loss": 0.8163,
+      "mean_token_accuracy": 0.7860001001507044,
+      "num_tokens": 274621990.0,
+      "step": 1570
+    },
+    {
+      "epoch": 0.43059837503619547,
+      "grad_norm": 0.828125,
+      "learning_rate": 4.04621435594887e-05,
+      "loss": 0.8319,
+      "mean_token_accuracy": 0.783361086063087,
+      "num_tokens": 276398989.0,
+      "step": 1580
+    },
+    {
+      "epoch": 0.43332368120731063,
+      "grad_norm": 0.8515625,
+      "learning_rate": 4.039190897597978e-05,
+      "loss": 0.7985,
+      "mean_token_accuracy": 0.7898858102038503,
+      "num_tokens": 278192884.0,
+      "step": 1590
+    },
+    {
+      "epoch": 0.4360489873784258,
+      "grad_norm": 0.9296875,
+      "learning_rate": 4.032167439247085e-05,
+      "loss": 0.8033,
+      "mean_token_accuracy": 0.7895436691120267,
+      "num_tokens": 279937931.0,
+      "step": 1600
+    },
+    {
+      "epoch": 0.43877429354954095,
+      "grad_norm": 0.75390625,
+      "learning_rate": 4.025143980896193e-05,
+      "loss": 0.7991,
+      "mean_token_accuracy": 0.7909117637202143,
+      "num_tokens": 281698143.0,
+      "step": 1610
+    },
+    {
+      "epoch": 0.4414995997206561,
+      "grad_norm": 0.79296875,
+      "learning_rate": 4.018120522545302e-05,
+      "loss": 0.8134,
+      "mean_token_accuracy": 0.7876318141818046,
+      "num_tokens": 283412565.0,
+      "step": 1620
+    },
+    {
+      "epoch": 0.44422490589177127,
+      "grad_norm": 0.90625,
+      "learning_rate": 4.011097064194409e-05,
+      "loss": 0.8195,
+      "mean_token_accuracy": 0.7865682732313871,
+      "num_tokens": 285184244.0,
+      "step": 1630
+    },
+    {
+      "epoch": 0.44695021206288643,
+      "grad_norm": 0.78515625,
+      "learning_rate": 4.004073605843517e-05,
+      "loss": 0.7748,
+      "mean_token_accuracy": 0.7957197193056345,
+      "num_tokens": 286963226.0,
+      "step": 1640
+    },
+    {
+      "epoch": 0.4496755182340016,
+      "grad_norm": 0.87890625,
+      "learning_rate": 3.997050147492625e-05,
+      "loss": 0.7904,
+      "mean_token_accuracy": 0.7924756951630115,
+      "num_tokens": 288742524.0,
+      "step": 1650
+    },
+    {
+      "epoch": 0.45240082440511675,
+      "grad_norm": 0.875,
+      "learning_rate": 3.990026689141734e-05,
+      "loss": 0.8196,
+      "mean_token_accuracy": 0.7863483810797334,
+      "num_tokens": 290486439.0,
+      "step": 1660
+    },
+    {
+      "epoch": 0.4551261305762319,
+      "grad_norm": 0.875,
+      "learning_rate": 3.9830032307908413e-05,
+      "loss": 0.827,
+      "mean_token_accuracy": 0.7851523524150252,
+      "num_tokens": 292244616.0,
+      "step": 1670
+    },
+    {
+      "epoch": 0.4578514367473471,
+      "grad_norm": 0.90234375,
+      "learning_rate": 3.9759797724399494e-05,
+      "loss": 0.8235,
+      "mean_token_accuracy": 0.7857779396697879,
+      "num_tokens": 294028863.0,
+      "step": 1680
+    },
+    {
+      "epoch": 0.46057674291846223,
+      "grad_norm": 0.8046875,
+      "learning_rate": 3.968956314089058e-05,
+      "loss": 0.7975,
+      "mean_token_accuracy": 0.7898599920794368,
+      "num_tokens": 295842660.0,
+      "step": 1690
+    },
+    {
+      "epoch": 0.4633020490895774,
+      "grad_norm": 0.80859375,
+      "learning_rate": 3.961932855738166e-05,
+      "loss": 0.8017,
+      "mean_token_accuracy": 0.7907447133213281,
+      "num_tokens": 297606076.0,
+      "step": 1700
+    },
+    {
+      "epoch": 0.46602735526069256,
+      "grad_norm": 0.84765625,
+      "learning_rate": 3.9549093973872734e-05,
+      "loss": 0.8289,
+      "mean_token_accuracy": 0.7834768177941441,
+      "num_tokens": 299367632.0,
+      "step": 1710
+    },
+    {
+      "epoch": 0.4687526614318077,
+      "grad_norm": 0.734375,
+      "learning_rate": 3.9478859390363814e-05,
+      "loss": 0.7901,
+      "mean_token_accuracy": 0.7932142606005073,
+      "num_tokens": 301097990.0,
+      "step": 1720
+    },
+    {
+      "epoch": 0.4714779676029229,
+      "grad_norm": 0.87109375,
+      "learning_rate": 3.94086248068549e-05,
+      "loss": 0.7945,
+      "mean_token_accuracy": 0.7922264555469155,
+      "num_tokens": 302847308.0,
+      "step": 1730
+    },
+    {
+      "epoch": 0.47420327377403804,
+      "grad_norm": 0.84375,
+      "learning_rate": 3.9338390223345975e-05,
+      "loss": 0.814,
+      "mean_token_accuracy": 0.7868275344371796,
+      "num_tokens": 304582625.0,
+      "step": 1740
+    },
+    {
+      "epoch": 0.4769285799451532,
+      "grad_norm": 0.79296875,
+      "learning_rate": 3.9268155639837055e-05,
+      "loss": 0.7997,
+      "mean_token_accuracy": 0.7915356256067753,
+      "num_tokens": 306358206.0,
+      "step": 1750
+    },
+    {
+      "epoch": 0.47965388611626836,
+      "grad_norm": 0.8828125,
+      "learning_rate": 3.9197921056328135e-05,
+      "loss": 0.8014,
+      "mean_token_accuracy": 0.7900160076096654,
+      "num_tokens": 308026562.0,
+      "step": 1760
+    },
+    {
+      "epoch": 0.4823791922873835,
+      "grad_norm": 0.85546875,
+      "learning_rate": 3.912768647281922e-05,
+      "loss": 0.8301,
+      "mean_token_accuracy": 0.7838495513424277,
+      "num_tokens": 309804543.0,
+      "step": 1770
+    },
+    {
+      "epoch": 0.4851044984584987,
+      "grad_norm": 0.84375,
+      "learning_rate": 3.9057451889310296e-05,
+      "loss": 0.8178,
+      "mean_token_accuracy": 0.7858808506280184,
+      "num_tokens": 311549771.0,
+      "step": 1780
+    },
+    {
+      "epoch": 0.48782980462961384,
+      "grad_norm": 0.8671875,
+      "learning_rate": 3.8987217305801376e-05,
+      "loss": 0.8295,
+      "mean_token_accuracy": 0.7843158490955829,
+      "num_tokens": 313326889.0,
+      "step": 1790
+    },
+    {
+      "epoch": 0.490555110800729,
+      "grad_norm": 0.80859375,
+      "learning_rate": 3.891698272229246e-05,
+      "loss": 0.7946,
+      "mean_token_accuracy": 0.7926847299560904,
+      "num_tokens": 315064715.0,
+      "step": 1800
+    },
+    {
+      "epoch": 0.49328041697184416,
+      "grad_norm": 0.8046875,
+      "learning_rate": 3.884674813878354e-05,
+      "loss": 0.8125,
+      "mean_token_accuracy": 0.7870695056393743,
+      "num_tokens": 316809509.0,
+      "step": 1810
+    },
+    {
+      "epoch": 0.4960057231429593,
+      "grad_norm": 0.80078125,
+      "learning_rate": 3.8776513555274616e-05,
+      "loss": 0.7918,
+      "mean_token_accuracy": 0.7922536037862301,
+      "num_tokens": 318597481.0,
+      "step": 1820
+    },
+    {
+      "epoch": 0.4987310293140745,
+      "grad_norm": 0.8125,
+      "learning_rate": 3.8706278971765697e-05,
+      "loss": 0.7964,
+      "mean_token_accuracy": 0.7912574496120215,
+      "num_tokens": 320382863.0,
+      "step": 1830
+    },
+    {
+      "epoch": 0.5014563354851896,
+      "grad_norm": 0.8125,
+      "learning_rate": 3.8636044388256784e-05,
+      "loss": 0.8057,
+      "mean_token_accuracy": 0.7904583893716335,
+      "num_tokens": 322136431.0,
+      "step": 1840
+    },
+    {
+      "epoch": 0.5041816416563049,
+      "grad_norm": 0.82421875,
+      "learning_rate": 3.856580980474786e-05,
+      "loss": 0.7952,
+      "mean_token_accuracy": 0.7914975183084607,
+      "num_tokens": 323892089.0,
+      "step": 1850
+    },
+    {
+      "epoch": 0.50690694782742,
+      "grad_norm": 0.76171875,
+      "learning_rate": 3.849557522123894e-05,
+      "loss": 0.7998,
+      "mean_token_accuracy": 0.7900273201987147,
+      "num_tokens": 325693457.0,
+      "step": 1860
+    },
+    {
+      "epoch": 0.5096322539985352,
+      "grad_norm": 0.89453125,
+      "learning_rate": 3.842534063773002e-05,
+      "loss": 0.7787,
+      "mean_token_accuracy": 0.7937103092670441,
+      "num_tokens": 327397884.0,
+      "step": 1870
+    },
+    {
+      "epoch": 0.5123575601696503,
+      "grad_norm": 0.90234375,
+      "learning_rate": 3.8355106054221104e-05,
+      "loss": 0.8215,
+      "mean_token_accuracy": 0.7870171546936036,
+      "num_tokens": 329077502.0,
+      "step": 1880
+    },
+    {
+      "epoch": 0.5150828663407655,
+      "grad_norm": 0.84765625,
+      "learning_rate": 3.828487147071218e-05,
+      "loss": 0.8104,
+      "mean_token_accuracy": 0.787852893024683,
+      "num_tokens": 330847695.0,
+      "step": 1890
+    },
+    {
+      "epoch": 0.5178081725118806,
+      "grad_norm": 0.8046875,
+      "learning_rate": 3.821463688720326e-05,
+      "loss": 0.7988,
+      "mean_token_accuracy": 0.7888111146166921,
+      "num_tokens": 332578366.0,
+      "step": 1900
+    },
+    {
+      "epoch": 0.5205334786829958,
+      "grad_norm": 0.85546875,
+      "learning_rate": 3.8144402303694345e-05,
+      "loss": 0.8188,
+      "mean_token_accuracy": 0.7861749952659011,
+      "num_tokens": 334293589.0,
+      "step": 1910
+    },
+    {
+      "epoch": 0.5232587848541109,
+      "grad_norm": 0.74609375,
+      "learning_rate": 3.8074167720185425e-05,
+      "loss": 0.8097,
+      "mean_token_accuracy": 0.7862037999555469,
+      "num_tokens": 336016972.0,
+      "step": 1920
+    },
+    {
+      "epoch": 0.5259840910252261,
+      "grad_norm": 0.765625,
+      "learning_rate": 3.80039331366765e-05,
+      "loss": 0.7887,
+      "mean_token_accuracy": 0.7920830117538571,
+      "num_tokens": 337805977.0,
+      "step": 1930
+    },
+    {
+      "epoch": 0.5287093971963412,
+      "grad_norm": 0.76171875,
+      "learning_rate": 3.793369855316758e-05,
+      "loss": 0.7668,
+      "mean_token_accuracy": 0.7962671579793096,
+      "num_tokens": 339548221.0,
+      "step": 1940
+    },
+    {
+      "epoch": 0.5314347033674565,
+      "grad_norm": 0.83984375,
+      "learning_rate": 3.7863463969658666e-05,
+      "loss": 0.835,
+      "mean_token_accuracy": 0.7823515819385648,
+      "num_tokens": 341268344.0,
+      "step": 1950
+    },
+    {
+      "epoch": 0.5341600095385716,
+      "grad_norm": 0.9296875,
+      "learning_rate": 3.779322938614974e-05,
+      "loss": 0.8047,
+      "mean_token_accuracy": 0.7884774927049876,
+      "num_tokens": 343017522.0,
+      "step": 1960
+    },
+    {
+      "epoch": 0.5368853157096868,
+      "grad_norm": 0.8515625,
+      "learning_rate": 3.772299480264082e-05,
+      "loss": 0.8029,
+      "mean_token_accuracy": 0.7902018539607525,
+      "num_tokens": 344699192.0,
+      "step": 1970
+    },
+    {
+      "epoch": 0.5396106218808019,
+      "grad_norm": 0.7734375,
+      "learning_rate": 3.7652760219131906e-05,
+      "loss": 0.7902,
+      "mean_token_accuracy": 0.7927109841257334,
+      "num_tokens": 346458527.0,
+      "step": 1980
+    },
+    {
+      "epoch": 0.5423359280519171,
+      "grad_norm": 0.8203125,
+      "learning_rate": 3.7582525635622986e-05,
+      "loss": 0.8071,
+      "mean_token_accuracy": 0.7881743613630533,
+      "num_tokens": 348204585.0,
+      "step": 1990
+    },
+    {
+      "epoch": 0.5450612342230322,
+      "grad_norm": 0.84375,
+      "learning_rate": 3.751229105211406e-05,
+      "loss": 0.8307,
+      "mean_token_accuracy": 0.7840012159198523,
+      "num_tokens": 349928259.0,
+      "step": 2000
+    },
+    {
+      "epoch": 0.5477865403941474,
+      "grad_norm": 0.83984375,
+      "learning_rate": 3.744205646860514e-05,
+      "loss": 0.7986,
+      "mean_token_accuracy": 0.7897524392232299,
+      "num_tokens": 351568018.0,
+      "step": 2010
+    },
+    {
+      "epoch": 0.5505118465652625,
+      "grad_norm": 0.8359375,
+      "learning_rate": 3.737182188509623e-05,
+      "loss": 0.7817,
+      "mean_token_accuracy": 0.7942662235349417,
+      "num_tokens": 353325716.0,
+      "step": 2020
+    },
+    {
+      "epoch": 0.5532371527363777,
+      "grad_norm": 0.84765625,
+      "learning_rate": 3.730158730158731e-05,
+      "loss": 0.8162,
+      "mean_token_accuracy": 0.7866754405200481,
+      "num_tokens": 355099655.0,
+      "step": 2030
+    },
+    {
+      "epoch": 0.5559624589074929,
+      "grad_norm": 0.7265625,
+      "learning_rate": 3.723135271807838e-05,
+      "loss": 0.798,
+      "mean_token_accuracy": 0.7905311169102788,
+      "num_tokens": 356892448.0,
+      "step": 2040
+    },
+    {
+      "epoch": 0.5586877650786081,
+      "grad_norm": 0.8359375,
+      "learning_rate": 3.716111813456946e-05,
+      "loss": 0.8193,
+      "mean_token_accuracy": 0.7858335379511118,
+      "num_tokens": 358599855.0,
+      "step": 2050
+    },
+    {
+      "epoch": 0.5614130712497232,
+      "grad_norm": 0.80859375,
+      "learning_rate": 3.709088355106055e-05,
+      "loss": 0.8256,
+      "mean_token_accuracy": 0.7851507473737002,
+      "num_tokens": 360366528.0,
+      "step": 2060
+    },
+    {
+      "epoch": 0.5641383774208384,
+      "grad_norm": 0.85546875,
+      "learning_rate": 3.702064896755162e-05,
+      "loss": 0.798,
+      "mean_token_accuracy": 0.7907180625945329,
+      "num_tokens": 362123903.0,
+      "step": 2070
+    },
+    {
+      "epoch": 0.5668636835919535,
+      "grad_norm": 0.7421875,
+      "learning_rate": 3.69504143840427e-05,
+      "loss": 0.7858,
+      "mean_token_accuracy": 0.7940833004191518,
+      "num_tokens": 363907270.0,
+      "step": 2080
+    },
+    {
+      "epoch": 0.5695889897630687,
+      "grad_norm": 0.94921875,
+      "learning_rate": 3.688017980053379e-05,
+      "loss": 0.7691,
+      "mean_token_accuracy": 0.7957627860829234,
+      "num_tokens": 365602883.0,
+      "step": 2090
+    },
+    {
+      "epoch": 0.5723142959341838,
+      "grad_norm": 0.8828125,
+      "learning_rate": 3.680994521702487e-05,
+      "loss": 0.7952,
+      "mean_token_accuracy": 0.7910092758014798,
+      "num_tokens": 367308921.0,
+      "step": 2100
+    },
+    {
+      "epoch": 0.575039602105299,
+      "grad_norm": 0.890625,
+      "learning_rate": 3.673971063351594e-05,
+      "loss": 0.7923,
+      "mean_token_accuracy": 0.7908395666629076,
+      "num_tokens": 369068618.0,
+      "step": 2110
+    },
+    {
+      "epoch": 0.5777649082764141,
+      "grad_norm": 0.7890625,
+      "learning_rate": 3.666947605000702e-05,
+      "loss": 0.8099,
+      "mean_token_accuracy": 0.7892173976637423,
+      "num_tokens": 370792272.0,
+      "step": 2120
+    },
+    {
+      "epoch": 0.5804902144475294,
+      "grad_norm": 0.86328125,
+      "learning_rate": 3.659924146649811e-05,
+      "loss": 0.8138,
+      "mean_token_accuracy": 0.7867964642122388,
+      "num_tokens": 372506541.0,
+      "step": 2130
+    },
+    {
+      "epoch": 0.5832155206186445,
+      "grad_norm": 0.80078125,
+      "learning_rate": 3.652900688298919e-05,
+      "loss": 0.7675,
+      "mean_token_accuracy": 0.7970227358862758,
+      "num_tokens": 374308542.0,
+      "step": 2140
+    },
+    {
+      "epoch": 0.5859408267897597,
+      "grad_norm": 0.82421875,
+      "learning_rate": 3.645877229948026e-05,
+      "loss": 0.7742,
+      "mean_token_accuracy": 0.7951750526204705,
+      "num_tokens": 376016266.0,
+      "step": 2150
+    },
+    {
+      "epoch": 0.5886661329608748,
+      "grad_norm": 0.78515625,
+      "learning_rate": 3.638853771597134e-05,
+      "loss": 0.8161,
+      "mean_token_accuracy": 0.7876632073894143,
+      "num_tokens": 377732713.0,
+      "step": 2160
+    },
+    {
+      "epoch": 0.59139143913199,
+      "grad_norm": 0.9296875,
+      "learning_rate": 3.631830313246243e-05,
+      "loss": 0.8051,
+      "mean_token_accuracy": 0.7895578004419803,
+      "num_tokens": 379457927.0,
+      "step": 2170
+    },
+    {
+      "epoch": 0.5941167453031051,
+      "grad_norm": 0.70703125,
+      "learning_rate": 3.6248068548953503e-05,
+      "loss": 0.8347,
+      "mean_token_accuracy": 0.7827049516141414,
+      "num_tokens": 381311531.0,
+      "step": 2180
+    },
+    {
+      "epoch": 0.5968420514742203,
+      "grad_norm": 0.8203125,
+      "learning_rate": 3.6177833965444584e-05,
+      "loss": 0.8261,
+      "mean_token_accuracy": 0.7865129733458162,
+      "num_tokens": 383076036.0,
+      "step": 2190
+    },
+    {
+      "epoch": 0.5995673576453354,
+      "grad_norm": 0.85546875,
+      "learning_rate": 3.610759938193567e-05,
+      "loss": 0.8222,
+      "mean_token_accuracy": 0.7858507174998521,
+      "num_tokens": 384869981.0,
+      "step": 2200
+    },
+    {
+      "epoch": 0.6022926638164506,
+      "grad_norm": 0.89453125,
+      "learning_rate": 3.603736479842675e-05,
+      "loss": 0.8048,
+      "mean_token_accuracy": 0.7886111721396446,
+      "num_tokens": 386654361.0,
+      "step": 2210
+    },
+    {
+      "epoch": 0.6050179699875657,
+      "grad_norm": 0.82421875,
+      "learning_rate": 3.5967130214917824e-05,
+      "loss": 0.7833,
+      "mean_token_accuracy": 0.7933692125603556,
+      "num_tokens": 388405388.0,
+      "step": 2220
+    },
+    {
+      "epoch": 0.607743276158681,
+      "grad_norm": 0.84375,
+      "learning_rate": 3.5896895631408904e-05,
+      "loss": 0.8132,
+      "mean_token_accuracy": 0.7873527022078634,
+      "num_tokens": 390160411.0,
+      "step": 2230
+    },
+    {
+      "epoch": 0.6104685823297961,
+      "grad_norm": 0.8359375,
+      "learning_rate": 3.582666104789999e-05,
+      "loss": 0.8175,
+      "mean_token_accuracy": 0.7858733765780925,
+      "num_tokens": 391963393.0,
+      "step": 2240
+    },
+    {
+      "epoch": 0.6131938885009113,
+      "grad_norm": 0.796875,
+      "learning_rate": 3.575642646439107e-05,
+      "loss": 0.8175,
+      "mean_token_accuracy": 0.7862450096756219,
+      "num_tokens": 393719494.0,
+      "step": 2250
+    },
+    {
+      "epoch": 0.6159191946720264,
+      "grad_norm": 0.76953125,
+      "learning_rate": 3.5686191880882145e-05,
+      "loss": 0.8148,
+      "mean_token_accuracy": 0.7875563595443964,
+      "num_tokens": 395419867.0,
+      "step": 2260
+    },
+    {
+      "epoch": 0.6186445008431416,
+      "grad_norm": 0.765625,
+      "learning_rate": 3.5615957297373225e-05,
+      "loss": 0.793,
+      "mean_token_accuracy": 0.7916922463104129,
+      "num_tokens": 397164636.0,
+      "step": 2270
+    },
+    {
+      "epoch": 0.6213698070142567,
+      "grad_norm": 0.8359375,
+      "learning_rate": 3.554572271386431e-05,
+      "loss": 0.8252,
+      "mean_token_accuracy": 0.7840616326779127,
+      "num_tokens": 398897880.0,
+      "step": 2280
+    },
+    {
+      "epoch": 0.6240951131853719,
+      "grad_norm": 0.859375,
+      "learning_rate": 3.5475488130355386e-05,
+      "loss": 0.8087,
+      "mean_token_accuracy": 0.7881853306666017,
+      "num_tokens": 400668616.0,
+      "step": 2290
+    },
+    {
+      "epoch": 0.6268204193564871,
+      "grad_norm": 0.7890625,
+      "learning_rate": 3.5405253546846466e-05,
+      "loss": 0.8029,
+      "mean_token_accuracy": 0.789029803685844,
+      "num_tokens": 402379710.0,
+      "step": 2300
+    },
+    {
+      "epoch": 0.6295457255276022,
+      "grad_norm": 0.80859375,
+      "learning_rate": 3.533501896333755e-05,
+      "loss": 0.7968,
+      "mean_token_accuracy": 0.790169364400208,
+      "num_tokens": 404084345.0,
+      "step": 2310
+    },
+    {
+      "epoch": 0.6322710316987175,
+      "grad_norm": 0.83984375,
+      "learning_rate": 3.526478437982863e-05,
+      "loss": 0.7746,
+      "mean_token_accuracy": 0.7961994405835867,
+      "num_tokens": 405770321.0,
+      "step": 2320
+    },
+    {
+      "epoch": 0.6349963378698326,
+      "grad_norm": 0.8125,
+      "learning_rate": 3.5194549796319706e-05,
+      "loss": 0.795,
+      "mean_token_accuracy": 0.7905052756890655,
+      "num_tokens": 407492020.0,
+      "step": 2330
+    },
+    {
+      "epoch": 0.6377216440409478,
+      "grad_norm": 0.79296875,
+      "learning_rate": 3.5124315212810787e-05,
+      "loss": 0.7911,
+      "mean_token_accuracy": 0.7934078704565763,
+      "num_tokens": 409257115.0,
+      "step": 2340
+    },
+    {
+      "epoch": 0.6404469502120629,
+      "grad_norm": 0.8515625,
+      "learning_rate": 3.5054080629301874e-05,
+      "loss": 0.8133,
+      "mean_token_accuracy": 0.787336241826415,
+      "num_tokens": 410986071.0,
+      "step": 2350
+    },
+    {
+      "epoch": 0.6431722563831781,
+      "grad_norm": 0.828125,
+      "learning_rate": 3.4983846045792954e-05,
+      "loss": 0.796,
+      "mean_token_accuracy": 0.7906939420849085,
+      "num_tokens": 412763897.0,
+      "step": 2360
+    },
+    {
+      "epoch": 0.6458975625542932,
+      "grad_norm": 0.796875,
+      "learning_rate": 3.491361146228403e-05,
+      "loss": 0.7717,
+      "mean_token_accuracy": 0.7971045158803463,
+      "num_tokens": 414503168.0,
+      "step": 2370
+    },
+    {
+      "epoch": 0.6486228687254084,
+      "grad_norm": 0.77734375,
+      "learning_rate": 3.484337687877511e-05,
+      "loss": 0.7931,
+      "mean_token_accuracy": 0.7926423370838165,
+      "num_tokens": 416227145.0,
+      "step": 2380
+    },
+    {
+      "epoch": 0.6513481748965235,
+      "grad_norm": 0.8203125,
+      "learning_rate": 3.4773142295266194e-05,
+      "loss": 0.7914,
+      "mean_token_accuracy": 0.7914676820859313,
+      "num_tokens": 417973040.0,
+      "step": 2390
+    },
+    {
+      "epoch": 0.6540734810676387,
+      "grad_norm": 0.87890625,
+      "learning_rate": 3.470290771175727e-05,
+      "loss": 0.7953,
+      "mean_token_accuracy": 0.790719479881227,
+      "num_tokens": 419736780.0,
+      "step": 2400
+    },
+    {
+      "epoch": 0.6567987872387538,
+      "grad_norm": 0.75390625,
+      "learning_rate": 3.463267312824835e-05,
+      "loss": 0.7871,
+      "mean_token_accuracy": 0.7933276878669858,
+      "num_tokens": 421516023.0,
+      "step": 2410
+    },
+    {
+      "epoch": 0.6595240934098691,
+      "grad_norm": 0.8359375,
+      "learning_rate": 3.4562438544739435e-05,
+      "loss": 0.824,
+      "mean_token_accuracy": 0.7856843816116452,
+      "num_tokens": 423324296.0,
+      "step": 2420
+    },
+    {
+      "epoch": 0.6622493995809842,
+      "grad_norm": 0.85546875,
+      "learning_rate": 3.4492203961230515e-05,
+      "loss": 0.814,
+      "mean_token_accuracy": 0.7854735240340233,
+      "num_tokens": 425114987.0,
+      "step": 2430
+    },
+    {
+      "epoch": 0.6649747057520994,
+      "grad_norm": 0.8125,
+      "learning_rate": 3.442196937772159e-05,
+      "loss": 0.7855,
+      "mean_token_accuracy": 0.7930196771398187,
+      "num_tokens": 426875751.0,
+      "step": 2440
+    },
+    {
+      "epoch": 0.6677000119232145,
+      "grad_norm": 0.80859375,
+      "learning_rate": 3.435173479421267e-05,
+      "loss": 0.7906,
+      "mean_token_accuracy": 0.7919770196080208,
+      "num_tokens": 428715433.0,
+      "step": 2450
+    },
+    {
+      "epoch": 0.6704253180943297,
+      "grad_norm": 0.953125,
+      "learning_rate": 3.4281500210703756e-05,
+      "loss": 0.829,
+      "mean_token_accuracy": 0.7838571049273014,
+      "num_tokens": 430443301.0,
+      "step": 2460
+    },
+    {
+      "epoch": 0.6731506242654448,
+      "grad_norm": 0.79296875,
+      "learning_rate": 3.4211265627194836e-05,
+      "loss": 0.7613,
+      "mean_token_accuracy": 0.7981071012094617,
+      "num_tokens": 432149898.0,
+      "step": 2470
+    },
+    {
+      "epoch": 0.67587593043656,
+      "grad_norm": 0.8515625,
+      "learning_rate": 3.414103104368591e-05,
+      "loss": 0.7814,
+      "mean_token_accuracy": 0.7945473201572895,
+      "num_tokens": 433904775.0,
+      "step": 2480
+    },
+    {
+      "epoch": 0.6786012366076751,
+      "grad_norm": 0.8359375,
+      "learning_rate": 3.407079646017699e-05,
+      "loss": 0.7814,
+      "mean_token_accuracy": 0.7934504115954042,
+      "num_tokens": 435648944.0,
+      "step": 2490
+    },
+    {
+      "epoch": 0.6813265427787903,
+      "grad_norm": 0.8828125,
+      "learning_rate": 3.4000561876668076e-05,
+      "loss": 0.8145,
+      "mean_token_accuracy": 0.7875463077798486,
+      "num_tokens": 437390122.0,
+      "step": 2500
+    },
+    {
+      "epoch": 0.6840518489499054,
+      "grad_norm": 0.83203125,
+      "learning_rate": 3.393032729315915e-05,
+      "loss": 0.8029,
+      "mean_token_accuracy": 0.7897700535133481,
+      "num_tokens": 439126311.0,
+      "step": 2510
+    },
+    {
+      "epoch": 0.6867771551210207,
+      "grad_norm": 0.80078125,
+      "learning_rate": 3.386009270965023e-05,
+      "loss": 0.7956,
+      "mean_token_accuracy": 0.7916007783263922,
+      "num_tokens": 440890933.0,
+      "step": 2520
+    },
+    {
+      "epoch": 0.6895024612921358,
+      "grad_norm": 0.95703125,
+      "learning_rate": 3.378985812614132e-05,
+      "loss": 0.7831,
+      "mean_token_accuracy": 0.7938105506822467,
+      "num_tokens": 442611770.0,
+      "step": 2530
+    },
+    {
+      "epoch": 0.692227767463251,
+      "grad_norm": 0.80859375,
+      "learning_rate": 3.37196235426324e-05,
+      "loss": 0.786,
+      "mean_token_accuracy": 0.7931900242343545,
+      "num_tokens": 444358166.0,
+      "step": 2540
+    },
+    {
+      "epoch": 0.6949530736343661,
+      "grad_norm": 0.81640625,
+      "learning_rate": 3.364938895912347e-05,
+      "loss": 0.824,
+      "mean_token_accuracy": 0.7852000340819358,
+      "num_tokens": 446078790.0,
+      "step": 2550
+    },
+    {
+      "epoch": 0.6976783798054813,
+      "grad_norm": 0.91796875,
+      "learning_rate": 3.357915437561455e-05,
+      "loss": 0.7918,
+      "mean_token_accuracy": 0.7913762830197811,
+      "num_tokens": 447811186.0,
+      "step": 2560
+    },
+    {
+      "epoch": 0.7004036859765964,
+      "grad_norm": 0.80078125,
+      "learning_rate": 3.350891979210564e-05,
+      "loss": 0.8043,
+      "mean_token_accuracy": 0.7889237821102142,
+      "num_tokens": 449537962.0,
+      "step": 2570
+    },
+    {
+      "epoch": 0.7031289921477116,
+      "grad_norm": 0.8125,
+      "learning_rate": 3.343868520859672e-05,
+      "loss": 0.7791,
+      "mean_token_accuracy": 0.7942595979198813,
+      "num_tokens": 451259469.0,
+      "step": 2580
+    },
+    {
+      "epoch": 0.7058542983188267,
+      "grad_norm": 0.8359375,
+      "learning_rate": 3.336845062508779e-05,
+      "loss": 0.7837,
+      "mean_token_accuracy": 0.7944825999438763,
+      "num_tokens": 453041146.0,
+      "step": 2590
+    },
+    {
+      "epoch": 0.708579604489942,
+      "grad_norm": 0.87109375,
+      "learning_rate": 3.329821604157887e-05,
+      "loss": 0.8168,
+      "mean_token_accuracy": 0.7870149469003082,
+      "num_tokens": 454753651.0,
+      "step": 2600
+    },
+    {
+      "epoch": 0.711304910661057,
+      "grad_norm": 0.83984375,
+      "learning_rate": 3.322798145806996e-05,
+      "loss": 0.7892,
+      "mean_token_accuracy": 0.7922771865502,
+      "num_tokens": 456537057.0,
+      "step": 2610
+    },
+    {
+      "epoch": 0.7140302168321723,
+      "grad_norm": 0.80078125,
+      "learning_rate": 3.315774687456103e-05,
+      "loss": 0.8299,
+      "mean_token_accuracy": 0.7850732276216149,
+      "num_tokens": 458271296.0,
+      "step": 2620
+    },
+    {
+      "epoch": 0.7167555230032874,
+      "grad_norm": 0.78125,
+      "learning_rate": 3.308751229105211e-05,
+      "loss": 0.8066,
+      "mean_token_accuracy": 0.7881610127165913,
+      "num_tokens": 460014932.0,
+      "step": 2630
+    },
+    {
+      "epoch": 0.7194808291744026,
+      "grad_norm": 0.890625,
+      "learning_rate": 3.30172777075432e-05,
+      "loss": 0.8148,
+      "mean_token_accuracy": 0.7869694575667381,
+      "num_tokens": 461818480.0,
+      "step": 2640
+    },
+    {
+      "epoch": 0.7222061353455177,
+      "grad_norm": 0.8671875,
+      "learning_rate": 3.294704312403428e-05,
+      "loss": 0.833,
+      "mean_token_accuracy": 0.7822133732959629,
+      "num_tokens": 463519993.0,
+      "step": 2650
+    },
+    {
+      "epoch": 0.7249314415166329,
+      "grad_norm": 0.90234375,
+      "learning_rate": 3.287680854052535e-05,
+      "loss": 0.8017,
+      "mean_token_accuracy": 0.7894423922523857,
+      "num_tokens": 465333101.0,
+      "step": 2660
+    },
+    {
+      "epoch": 0.727656747687748,
+      "grad_norm": 0.82421875,
+      "learning_rate": 3.280657395701643e-05,
+      "loss": 0.7965,
+      "mean_token_accuracy": 0.7905446164309978,
+      "num_tokens": 467047198.0,
+      "step": 2670
+    },
+    {
+      "epoch": 0.7303820538588632,
+      "grad_norm": 0.83984375,
+      "learning_rate": 3.273633937350752e-05,
+      "loss": 0.8101,
+      "mean_token_accuracy": 0.7884848836809397,
+      "num_tokens": 468806711.0,
+      "step": 2680
+    },
+    {
+      "epoch": 0.7331073600299783,
+      "grad_norm": 0.80078125,
+      "learning_rate": 3.26661047899986e-05,
+      "loss": 0.7703,
+      "mean_token_accuracy": 0.7962682554498315,
+      "num_tokens": 470600782.0,
+      "step": 2690
+    },
+    {
+      "epoch": 0.7358326662010936,
+      "grad_norm": 0.8125,
+      "learning_rate": 3.2595870206489674e-05,
+      "loss": 0.8074,
+      "mean_token_accuracy": 0.7884658765047788,
+      "num_tokens": 472349330.0,
+      "step": 2700
+    },
+    {
+      "epoch": 0.7385579723722087,
+      "grad_norm": 0.81640625,
+      "learning_rate": 3.2525635622980754e-05,
+      "loss": 0.7991,
+      "mean_token_accuracy": 0.7914090482518077,
+      "num_tokens": 474106983.0,
+      "step": 2710
+    },
+    {
+      "epoch": 0.7412832785433239,
+      "grad_norm": 0.86328125,
+      "learning_rate": 3.245540103947184e-05,
+      "loss": 0.8247,
+      "mean_token_accuracy": 0.7848868206143379,
+      "num_tokens": 475902407.0,
+      "step": 2720
+    },
+    {
+      "epoch": 0.744008584714439,
+      "grad_norm": 0.78515625,
+      "learning_rate": 3.2385166455962914e-05,
+      "loss": 0.7592,
+      "mean_token_accuracy": 0.7984217805787921,
+      "num_tokens": 477654644.0,
+      "step": 2730
+    },
+    {
+      "epoch": 0.7467338908855542,
+      "grad_norm": 0.83984375,
+      "learning_rate": 3.2314931872453994e-05,
+      "loss": 0.8171,
+      "mean_token_accuracy": 0.7861290844157338,
+      "num_tokens": 479409078.0,
+      "step": 2740
+    },
+    {
+      "epoch": 0.7494591970566693,
+      "grad_norm": 0.96484375,
+      "learning_rate": 3.224469728894508e-05,
+      "loss": 0.8034,
+      "mean_token_accuracy": 0.789521318115294,
+      "num_tokens": 481154972.0,
+      "step": 2750
+    },
+    {
+      "epoch": 0.7521845032277845,
+      "grad_norm": 0.859375,
+      "learning_rate": 3.217446270543616e-05,
+      "loss": 0.8281,
+      "mean_token_accuracy": 0.7843747129663825,
+      "num_tokens": 482957554.0,
+      "step": 2760
+    },
+    {
+      "epoch": 0.7549098093988996,
+      "grad_norm": 0.82421875,
+      "learning_rate": 3.2104228121927235e-05,
+      "loss": 0.8035,
+      "mean_token_accuracy": 0.7887431923300028,
+      "num_tokens": 484730289.0,
+      "step": 2770
+    },
+    {
+      "epoch": 0.7576351155700148,
+      "grad_norm": 0.9140625,
+      "learning_rate": 3.2033993538418315e-05,
+      "loss": 0.81,
+      "mean_token_accuracy": 0.7866667127236724,
+      "num_tokens": 486390658.0,
+      "step": 2780
+    },
+    {
+      "epoch": 0.7603604217411299,
+      "grad_norm": 0.890625,
+      "learning_rate": 3.19637589549094e-05,
+      "loss": 0.7721,
+      "mean_token_accuracy": 0.7958020122721792,
+      "num_tokens": 488084357.0,
+      "step": 2790
+    },
+    {
+      "epoch": 0.7630857279122452,
+      "grad_norm": 0.8984375,
+      "learning_rate": 3.189352437140048e-05,
+      "loss": 0.8449,
+      "mean_token_accuracy": 0.7812584444880486,
+      "num_tokens": 489867504.0,
+      "step": 2800
+    },
+    {
+      "epoch": 0.7658110340833603,
+      "grad_norm": 0.80078125,
+      "learning_rate": 3.1823289787891556e-05,
+      "loss": 0.8125,
+      "mean_token_accuracy": 0.786544431746006,
+      "num_tokens": 491614120.0,
+      "step": 2810
+    },
+    {
+      "epoch": 0.7685363402544755,
+      "grad_norm": 0.76171875,
+      "learning_rate": 3.1753055204382636e-05,
+      "loss": 0.8118,
+      "mean_token_accuracy": 0.7891720440238714,
+      "num_tokens": 493360108.0,
+      "step": 2820
+    },
+    {
+      "epoch": 0.7712616464255906,
+      "grad_norm": 0.84375,
+      "learning_rate": 3.168282062087372e-05,
+      "loss": 0.7934,
+      "mean_token_accuracy": 0.7931292155757547,
+      "num_tokens": 495060867.0,
+      "step": 2830
+    },
+    {
+      "epoch": 0.7739869525967058,
+      "grad_norm": 0.859375,
+      "learning_rate": 3.16125860373648e-05,
+      "loss": 0.8128,
+      "mean_token_accuracy": 0.786138348095119,
+      "num_tokens": 496731129.0,
+      "step": 2840
+    },
+    {
+      "epoch": 0.7767122587678209,
+      "grad_norm": 0.76171875,
+      "learning_rate": 3.1542351453855877e-05,
+      "loss": 0.7858,
+      "mean_token_accuracy": 0.7930129086598754,
+      "num_tokens": 498416929.0,
+      "step": 2850
+    },
+    {
+      "epoch": 0.7794375649389361,
+      "grad_norm": 0.85546875,
+      "learning_rate": 3.1472116870346964e-05,
+      "loss": 0.7707,
+      "mean_token_accuracy": 0.7968917129561305,
+      "num_tokens": 500131030.0,
+      "step": 2860
+    },
+    {
+      "epoch": 0.7821628711100512,
+      "grad_norm": 0.765625,
+      "learning_rate": 3.1401882286838044e-05,
+      "loss": 0.7833,
+      "mean_token_accuracy": 0.7935072341933846,
+      "num_tokens": 501860254.0,
+      "step": 2870
+    },
+    {
+      "epoch": 0.7848881772811664,
+      "grad_norm": 0.83984375,
+      "learning_rate": 3.133164770332912e-05,
+      "loss": 0.8296,
+      "mean_token_accuracy": 0.7837417518720031,
+      "num_tokens": 503599791.0,
+      "step": 2880
+    },
+    {
+      "epoch": 0.7876134834522815,
+      "grad_norm": 0.796875,
+      "learning_rate": 3.12614131198202e-05,
+      "loss": 0.8438,
+      "mean_token_accuracy": 0.7812881203368306,
+      "num_tokens": 505277304.0,
+      "step": 2890
+    },
+    {
+      "epoch": 0.7903387896233968,
+      "grad_norm": 0.8828125,
+      "learning_rate": 3.1191178536311284e-05,
+      "loss": 0.8007,
+      "mean_token_accuracy": 0.7895943874493241,
+      "num_tokens": 507010506.0,
+      "step": 2900
+    },
+    {
+      "epoch": 0.7930640957945119,
+      "grad_norm": 0.83984375,
+      "learning_rate": 3.1120943952802364e-05,
+      "loss": 0.7916,
+      "mean_token_accuracy": 0.7917691525071859,
+      "num_tokens": 508717678.0,
+      "step": 2910
+    },
+    {
+      "epoch": 0.7957894019656271,
+      "grad_norm": 0.8046875,
+      "learning_rate": 3.105070936929344e-05,
+      "loss": 0.7715,
+      "mean_token_accuracy": 0.7972906947135925,
+      "num_tokens": 510478982.0,
+      "step": 2920
+    },
+    {
+      "epoch": 0.7985147081367422,
+      "grad_norm": 0.80859375,
+      "learning_rate": 3.0980474785784525e-05,
+      "loss": 0.765,
+      "mean_token_accuracy": 0.7968467550352216,
+      "num_tokens": 512223492.0,
+      "step": 2930
+    },
+    {
+      "epoch": 0.8012400143078574,
+      "grad_norm": 0.7890625,
+      "learning_rate": 3.0910240202275605e-05,
+      "loss": 0.7869,
+      "mean_token_accuracy": 0.7923903580754995,
+      "num_tokens": 513950826.0,
+      "step": 2940
+    },
+    {
+      "epoch": 0.8039653204789725,
+      "grad_norm": 0.83203125,
+      "learning_rate": 3.0840005618766685e-05,
+      "loss": 0.8145,
+      "mean_token_accuracy": 0.7880570556037128,
+      "num_tokens": 515747512.0,
+      "step": 2950
+    },
+    {
+      "epoch": 0.8066906266500877,
+      "grad_norm": 0.78125,
+      "learning_rate": 3.076977103525776e-05,
+      "loss": 0.7891,
+      "mean_token_accuracy": 0.7921149514615535,
+      "num_tokens": 517564408.0,
+      "step": 2960
+    },
+    {
+      "epoch": 0.8094159328212028,
+      "grad_norm": 0.796875,
+      "learning_rate": 3.0699536451748846e-05,
+      "loss": 0.7381,
+      "mean_token_accuracy": 0.8031238542869688,
+      "num_tokens": 519335113.0,
+      "step": 2970
+    },
+    {
+      "epoch": 0.812141238992318,
+      "grad_norm": 0.76953125,
+      "learning_rate": 3.0629301868239926e-05,
+      "loss": 0.7765,
+      "mean_token_accuracy": 0.7950917206704616,
+      "num_tokens": 521052694.0,
+      "step": 2980
+    },
+    {
+      "epoch": 0.8148665451634332,
+      "grad_norm": 0.8203125,
+      "learning_rate": 3.0559067284731e-05,
+      "loss": 0.8083,
+      "mean_token_accuracy": 0.7879104994237422,
+      "num_tokens": 522838886.0,
+      "step": 2990
+    },
+    {
+      "epoch": 0.8175918513345484,
+      "grad_norm": 0.7578125,
+      "learning_rate": 3.048883270122208e-05,
+      "loss": 0.7942,
+      "mean_token_accuracy": 0.7910878494381904,
+      "num_tokens": 524659688.0,
+      "step": 3000
+    },
+    {
+      "epoch": 0.8203171575056635,
+      "grad_norm": 0.796875,
+      "learning_rate": 3.0418598117713166e-05,
+      "loss": 0.7813,
+      "mean_token_accuracy": 0.7939989395439625,
+      "num_tokens": 526428828.0,
+      "step": 3010
+    },
+    {
+      "epoch": 0.8230424636767787,
+      "grad_norm": 0.796875,
+      "learning_rate": 3.0348363534204243e-05,
+      "loss": 0.8012,
+      "mean_token_accuracy": 0.7889990432187914,
+      "num_tokens": 528178357.0,
+      "step": 3020
+    },
+    {
+      "epoch": 0.8257677698478938,
+      "grad_norm": 0.79296875,
+      "learning_rate": 3.0278128950695323e-05,
+      "loss": 0.7823,
+      "mean_token_accuracy": 0.793020773679018,
+      "num_tokens": 529968661.0,
+      "step": 3030
+    },
+    {
+      "epoch": 0.828493076019009,
+      "grad_norm": 0.875,
+      "learning_rate": 3.0207894367186407e-05,
+      "loss": 0.7936,
+      "mean_token_accuracy": 0.7926454781554639,
+      "num_tokens": 531735846.0,
+      "step": 3040
+    },
+    {
+      "epoch": 0.8312183821901241,
+      "grad_norm": 0.8515625,
+      "learning_rate": 3.0137659783677484e-05,
+      "loss": 0.8116,
+      "mean_token_accuracy": 0.7876941539347172,
+      "num_tokens": 533482605.0,
+      "step": 3050
+    },
+    {
+      "epoch": 0.8339436883612393,
+      "grad_norm": 0.796875,
+      "learning_rate": 3.0067425200168564e-05,
+      "loss": 0.7989,
+      "mean_token_accuracy": 0.7907701600342989,
+      "num_tokens": 535172117.0,
+      "step": 3060
+    },
+    {
+      "epoch": 0.8366689945323545,
+      "grad_norm": 0.7890625,
+      "learning_rate": 2.9997190616659644e-05,
+      "loss": 0.7873,
+      "mean_token_accuracy": 0.7930120758712291,
+      "num_tokens": 536922982.0,
+      "step": 3070
+    },
+    {
+      "epoch": 0.8393943007034697,
+      "grad_norm": 0.83203125,
+      "learning_rate": 2.9926956033150728e-05,
+      "loss": 0.7965,
+      "mean_token_accuracy": 0.7907984968274832,
+      "num_tokens": 538697561.0,
+      "step": 3080
+    },
+    {
+      "epoch": 0.8421196068745849,
+      "grad_norm": 0.90625,
+      "learning_rate": 2.9856721449641805e-05,
+      "loss": 0.8019,
+      "mean_token_accuracy": 0.7898768160492182,
+      "num_tokens": 540498809.0,
+      "step": 3090
+    },
+    {
+      "epoch": 0.8448449130457,
+      "grad_norm": 0.859375,
+      "learning_rate": 2.9786486866132885e-05,
+      "loss": 0.7751,
+      "mean_token_accuracy": 0.7947335474193096,
+      "num_tokens": 542220573.0,
+      "step": 3100
+    },
+    {
+      "epoch": 0.8475702192168152,
+      "grad_norm": 0.83203125,
+      "learning_rate": 2.971625228262396e-05,
+      "loss": 0.7912,
+      "mean_token_accuracy": 0.7921573128551245,
+      "num_tokens": 543995979.0,
+      "step": 3110
+    },
+    {
+      "epoch": 0.8502955253879303,
+      "grad_norm": 0.8046875,
+      "learning_rate": 2.964601769911505e-05,
+      "loss": 0.8045,
+      "mean_token_accuracy": 0.7876936543732882,
+      "num_tokens": 545762046.0,
+      "step": 3120
+    },
+    {
+      "epoch": 0.8530208315590455,
+      "grad_norm": 0.7421875,
+      "learning_rate": 2.9575783115606125e-05,
+      "loss": 0.7829,
+      "mean_token_accuracy": 0.7934223107993603,
+      "num_tokens": 547563120.0,
+      "step": 3130
+    },
+    {
+      "epoch": 0.8557461377301606,
+      "grad_norm": 0.828125,
+      "learning_rate": 2.9505548532097206e-05,
+      "loss": 0.802,
+      "mean_token_accuracy": 0.7895748918876052,
+      "num_tokens": 549276605.0,
+      "step": 3140
+    },
+    {
+      "epoch": 0.8584714439012758,
+      "grad_norm": 0.83203125,
+      "learning_rate": 2.943531394858829e-05,
+      "loss": 0.7957,
+      "mean_token_accuracy": 0.7906818810850382,
+      "num_tokens": 550991479.0,
+      "step": 3150
+    },
+    {
+      "epoch": 0.8611967500723909,
+      "grad_norm": 0.8203125,
+      "learning_rate": 2.9365079365079366e-05,
+      "loss": 0.7845,
+      "mean_token_accuracy": 0.7940855285152792,
+      "num_tokens": 552777662.0,
+      "step": 3160
+    },
+    {
+      "epoch": 0.8639220562435062,
+      "grad_norm": 0.93359375,
+      "learning_rate": 2.9294844781570446e-05,
+      "loss": 0.7724,
+      "mean_token_accuracy": 0.7957506142556667,
+      "num_tokens": 554539356.0,
+      "step": 3170
+    },
+    {
+      "epoch": 0.8666473624146213,
+      "grad_norm": 0.828125,
+      "learning_rate": 2.9224610198061526e-05,
+      "loss": 0.8331,
+      "mean_token_accuracy": 0.7827875496819615,
+      "num_tokens": 556212548.0,
+      "step": 3180
+    },
+    {
+      "epoch": 0.8693726685857365,
+      "grad_norm": 0.8515625,
+      "learning_rate": 2.915437561455261e-05,
+      "loss": 0.8014,
+      "mean_token_accuracy": 0.7891403412446379,
+      "num_tokens": 557950461.0,
+      "step": 3190
+    },
+    {
+      "epoch": 0.8720979747568516,
+      "grad_norm": 0.85546875,
+      "learning_rate": 2.9084141031043687e-05,
+      "loss": 0.7838,
+      "mean_token_accuracy": 0.7931245841085911,
+      "num_tokens": 559715926.0,
+      "step": 3200
+    },
+    {
+      "epoch": 0.8748232809279668,
+      "grad_norm": 0.88671875,
+      "learning_rate": 2.9013906447534767e-05,
+      "loss": 0.8002,
+      "mean_token_accuracy": 0.7911280466243624,
+      "num_tokens": 561479394.0,
+      "step": 3210
+    },
+    {
+      "epoch": 0.8775485870990819,
+      "grad_norm": 0.78515625,
+      "learning_rate": 2.8943671864025844e-05,
+      "loss": 0.7954,
+      "mean_token_accuracy": 0.7900624511763453,
+      "num_tokens": 563271872.0,
+      "step": 3220
+    },
+    {
+      "epoch": 0.8802738932701971,
+      "grad_norm": 0.78515625,
+      "learning_rate": 2.887343728051693e-05,
+      "loss": 0.7892,
+      "mean_token_accuracy": 0.7931271711364388,
+      "num_tokens": 565002210.0,
+      "step": 3230
+    },
+    {
+      "epoch": 0.8829991994413122,
+      "grad_norm": 0.87109375,
+      "learning_rate": 2.8803202697008008e-05,
+      "loss": 0.7221,
+      "mean_token_accuracy": 0.8066132063046098,
+      "num_tokens": 566740287.0,
+      "step": 3240
+    },
+    {
+      "epoch": 0.8857245056124274,
+      "grad_norm": 0.76171875,
+      "learning_rate": 2.8732968113499088e-05,
+      "loss": 0.819,
+      "mean_token_accuracy": 0.7862404658459127,
+      "num_tokens": 568560935.0,
+      "step": 3250
+    },
+    {
+      "epoch": 0.8884498117835425,
+      "grad_norm": 0.921875,
+      "learning_rate": 2.866273352999017e-05,
+      "loss": 0.8052,
+      "mean_token_accuracy": 0.7889881532639265,
+      "num_tokens": 570290159.0,
+      "step": 3260
+    },
+    {
+      "epoch": 0.8911751179546578,
+      "grad_norm": 0.8515625,
+      "learning_rate": 2.8592498946481248e-05,
+      "loss": 0.813,
+      "mean_token_accuracy": 0.7877601679414511,
+      "num_tokens": 572076704.0,
+      "step": 3270
+    },
+    {
+      "epoch": 0.8939004241257729,
+      "grad_norm": 0.7890625,
+      "learning_rate": 2.852226436297233e-05,
+      "loss": 0.8186,
+      "mean_token_accuracy": 0.7869238125160336,
+      "num_tokens": 573841975.0,
+      "step": 3280
+    },
+    {
+      "epoch": 0.8966257302968881,
+      "grad_norm": 0.7578125,
+      "learning_rate": 2.845202977946341e-05,
+      "loss": 0.7977,
+      "mean_token_accuracy": 0.7911882294341922,
+      "num_tokens": 575587035.0,
+      "step": 3290
+    },
+    {
+      "epoch": 0.8993510364680032,
+      "grad_norm": 0.77734375,
+      "learning_rate": 2.8381795195954492e-05,
+      "loss": 0.798,
+      "mean_token_accuracy": 0.7903127640485763,
+      "num_tokens": 577285852.0,
+      "step": 3300
+    },
+    {
+      "epoch": 0.9020763426391184,
+      "grad_norm": 0.76953125,
+      "learning_rate": 2.831156061244557e-05,
+      "loss": 0.8095,
+      "mean_token_accuracy": 0.7888897746801377,
+      "num_tokens": 579085053.0,
+      "step": 3310
+    },
+    {
+      "epoch": 0.9048016488102335,
+      "grad_norm": 0.74609375,
+      "learning_rate": 2.824132602893665e-05,
+      "loss": 0.7856,
+      "mean_token_accuracy": 0.7931374186649919,
+      "num_tokens": 580874041.0,
+      "step": 3320
+    },
+    {
+      "epoch": 0.9075269549813487,
+      "grad_norm": 0.84375,
+      "learning_rate": 2.8171091445427726e-05,
+      "loss": 0.783,
+      "mean_token_accuracy": 0.792279077321291,
+      "num_tokens": 582616757.0,
+      "step": 3330
+    },
+    {
+      "epoch": 0.9102522611524638,
+      "grad_norm": 0.80859375,
+      "learning_rate": 2.8100856861918813e-05,
+      "loss": 0.7776,
+      "mean_token_accuracy": 0.7949761841446161,
+      "num_tokens": 584370810.0,
+      "step": 3340
+    },
+    {
+      "epoch": 0.912977567323579,
+      "grad_norm": 0.8203125,
+      "learning_rate": 2.803062227840989e-05,
+      "loss": 0.7923,
+      "mean_token_accuracy": 0.7915155973285437,
+      "num_tokens": 586143330.0,
+      "step": 3350
+    },
+    {
+      "epoch": 0.9157028734946941,
+      "grad_norm": 0.7890625,
+      "learning_rate": 2.796038769490097e-05,
+      "loss": 0.7621,
+      "mean_token_accuracy": 0.7981992844492197,
+      "num_tokens": 587851884.0,
+      "step": 3360
+    },
+    {
+      "epoch": 0.9184281796658094,
+      "grad_norm": 0.83984375,
+      "learning_rate": 2.7890153111392054e-05,
+      "loss": 0.7864,
+      "mean_token_accuracy": 0.7917513215914369,
+      "num_tokens": 589601267.0,
+      "step": 3370
+    },
+    {
+      "epoch": 0.9211534858369245,
+      "grad_norm": 0.92578125,
+      "learning_rate": 2.7819918527883134e-05,
+      "loss": 0.8136,
+      "mean_token_accuracy": 0.7862017085775733,
+      "num_tokens": 591321205.0,
+      "step": 3380
+    },
+    {
+      "epoch": 0.9238787920080397,
+      "grad_norm": 0.73828125,
+      "learning_rate": 2.774968394437421e-05,
+      "loss": 0.7677,
+      "mean_token_accuracy": 0.797662234865129,
+      "num_tokens": 593110103.0,
+      "step": 3390
+    },
+    {
+      "epoch": 0.9266040981791548,
+      "grad_norm": 0.875,
+      "learning_rate": 2.767944936086529e-05,
+      "loss": 0.8164,
+      "mean_token_accuracy": 0.7869841903448105,
+      "num_tokens": 594823039.0,
+      "step": 3400
+    },
+    {
+      "epoch": 0.92932940435027,
+      "grad_norm": 0.78125,
+      "learning_rate": 2.7609214777356374e-05,
+      "loss": 0.7943,
+      "mean_token_accuracy": 0.79167689550668,
+      "num_tokens": 596625643.0,
+      "step": 3410
+    },
+    {
+      "epoch": 0.9320547105213851,
+      "grad_norm": 0.8203125,
+      "learning_rate": 2.753898019384745e-05,
+      "loss": 0.8081,
+      "mean_token_accuracy": 0.7864180063828826,
+      "num_tokens": 598409404.0,
+      "step": 3420
+    },
+    {
+      "epoch": 0.9347800166925003,
+      "grad_norm": 1.0703125,
+      "learning_rate": 2.746874561033853e-05,
+      "loss": 0.7998,
+      "mean_token_accuracy": 0.7896339423023164,
+      "num_tokens": 600131817.0,
+      "step": 3430
+    },
+    {
+      "epoch": 0.9375053228636154,
+      "grad_norm": 0.73046875,
+      "learning_rate": 2.7398511026829608e-05,
+      "loss": 0.8006,
+      "mean_token_accuracy": 0.7904914256185294,
+      "num_tokens": 601867823.0,
+      "step": 3440
+    },
+    {
+      "epoch": 0.9402306290347306,
+      "grad_norm": 0.8203125,
+      "learning_rate": 2.7328276443320695e-05,
+      "loss": 0.7787,
+      "mean_token_accuracy": 0.7962382351979613,
+      "num_tokens": 603591263.0,
+      "step": 3450
+    },
+    {
+      "epoch": 0.9429559352058458,
+      "grad_norm": 0.84765625,
+      "learning_rate": 2.7258041859811772e-05,
+      "loss": 0.8041,
+      "mean_token_accuracy": 0.7898189172148704,
+      "num_tokens": 605400422.0,
+      "step": 3460
+    },
+    {
+      "epoch": 0.945681241376961,
+      "grad_norm": 0.8203125,
+      "learning_rate": 2.7187807276302852e-05,
+      "loss": 0.7985,
+      "mean_token_accuracy": 0.7910171849653125,
+      "num_tokens": 607180330.0,
+      "step": 3470
+    },
+    {
+      "epoch": 0.9484065475480761,
+      "grad_norm": 0.75390625,
+      "learning_rate": 2.7117572692793936e-05,
+      "loss": 0.7705,
+      "mean_token_accuracy": 0.797027708683163,
+      "num_tokens": 608937194.0,
+      "step": 3480
+    },
+    {
+      "epoch": 0.9511318537191913,
+      "grad_norm": 0.83203125,
+      "learning_rate": 2.7047338109285016e-05,
+      "loss": 0.8022,
+      "mean_token_accuracy": 0.7885818719863892,
+      "num_tokens": 610676168.0,
+      "step": 3490
+    },
+    {
+      "epoch": 0.9538571598903064,
+      "grad_norm": 0.7890625,
+      "learning_rate": 2.6977103525776093e-05,
+      "loss": 0.7633,
+      "mean_token_accuracy": 0.79855783469975,
+      "num_tokens": 612491555.0,
+      "step": 3500
+    },
+    {
+      "epoch": 0.9565824660614216,
+      "grad_norm": 0.84765625,
+      "learning_rate": 2.6906868942267173e-05,
+      "loss": 0.8097,
+      "mean_token_accuracy": 0.7876760197803379,
+      "num_tokens": 614310820.0,
+      "step": 3510
+    },
+    {
+      "epoch": 0.9593077722325367,
+      "grad_norm": 0.79296875,
+      "learning_rate": 2.6836634358758256e-05,
+      "loss": 0.8102,
+      "mean_token_accuracy": 0.7879532830789685,
+      "num_tokens": 616071087.0,
+      "step": 3520
+    },
+    {
+      "epoch": 0.9620330784036519,
+      "grad_norm": 0.8671875,
+      "learning_rate": 2.6766399775249333e-05,
+      "loss": 0.7671,
+      "mean_token_accuracy": 0.7961735974997282,
+      "num_tokens": 617769166.0,
+      "step": 3530
+    },
+    {
+      "epoch": 0.964758384574767,
+      "grad_norm": 0.859375,
+      "learning_rate": 2.6696165191740413e-05,
+      "loss": 0.7933,
+      "mean_token_accuracy": 0.7911225285381078,
+      "num_tokens": 619538232.0,
+      "step": 3540
+    },
+    {
+      "epoch": 0.9674836907458823,
+      "grad_norm": 0.71875,
+      "learning_rate": 2.662593060823149e-05,
+      "loss": 0.7867,
+      "mean_token_accuracy": 0.7945138230919838,
+      "num_tokens": 621303314.0,
+      "step": 3550
+    },
+    {
+      "epoch": 0.9702089969169974,
+      "grad_norm": 0.74609375,
+      "learning_rate": 2.6555696024722577e-05,
+      "loss": 0.7743,
+      "mean_token_accuracy": 0.7957165893167257,
+      "num_tokens": 623100776.0,
+      "step": 3560
+    },
+    {
+      "epoch": 0.9729343030881126,
+      "grad_norm": 0.79296875,
+      "learning_rate": 2.6485461441213654e-05,
+      "loss": 0.8252,
+      "mean_token_accuracy": 0.7833328101783991,
+      "num_tokens": 624821413.0,
+      "step": 3570
+    },
+    {
+      "epoch": 0.9756596092592277,
+      "grad_norm": 0.8828125,
+      "learning_rate": 2.6415226857704734e-05,
+      "loss": 0.802,
+      "mean_token_accuracy": 0.7890756510198116,
+      "num_tokens": 626628545.0,
+      "step": 3580
+    },
+    {
+      "epoch": 0.9783849154303429,
+      "grad_norm": 0.7734375,
+      "learning_rate": 2.6344992274195818e-05,
+      "loss": 0.7839,
+      "mean_token_accuracy": 0.7937445532530546,
+      "num_tokens": 628425036.0,
+      "step": 3590
+    },
+    {
+      "epoch": 0.981110221601458,
+      "grad_norm": 0.76171875,
+      "learning_rate": 2.6274757690686898e-05,
+      "loss": 0.7585,
+      "mean_token_accuracy": 0.7974846661090851,
+      "num_tokens": 630092739.0,
+      "step": 3600
+    },
+    {
+      "epoch": 0.9838355277725732,
+      "grad_norm": 0.9453125,
+      "learning_rate": 2.6204523107177975e-05,
+      "loss": 0.8083,
+      "mean_token_accuracy": 0.7897484470158815,
+      "num_tokens": 631791605.0,
+      "step": 3610
+    },
+    {
+      "epoch": 0.9865608339436883,
+      "grad_norm": 0.8359375,
+      "learning_rate": 2.6134288523669055e-05,
+      "loss": 0.7971,
+      "mean_token_accuracy": 0.7893307173624635,
+      "num_tokens": 633554055.0,
+      "step": 3620
+    },
+    {
+      "epoch": 0.9892861401148035,
+      "grad_norm": 0.796875,
+      "learning_rate": 2.606405394016014e-05,
+      "loss": 0.7682,
+      "mean_token_accuracy": 0.7960259525105358,
+      "num_tokens": 635269789.0,
+      "step": 3630
+    },
+    {
+      "epoch": 0.9920114462859186,
+      "grad_norm": 0.79296875,
+      "learning_rate": 2.5993819356651215e-05,
+      "loss": 0.7834,
+      "mean_token_accuracy": 0.79379309322685,
+      "num_tokens": 637078092.0,
+      "step": 3640
+    },
+    {
+      "epoch": 0.9947367524570339,
+      "grad_norm": 0.83984375,
+      "learning_rate": 2.5923584773142296e-05,
+      "loss": 0.768,
+      "mean_token_accuracy": 0.798496849089861,
+      "num_tokens": 638813927.0,
+      "step": 3650
+    },
+    {
+      "epoch": 0.997462058628149,
+      "grad_norm": 0.91796875,
+      "learning_rate": 2.5853350189633372e-05,
+      "loss": 0.7887,
+      "mean_token_accuracy": 0.7931042012758553,
+      "num_tokens": 640594007.0,
+      "step": 3660
+    },
+    {
+      "epoch": 1.0,
+      "grad_norm": 0.435546875,
+      "learning_rate": 2.578311560612446e-05,
+      "loss": 0.7024,
+      "mean_token_accuracy": 0.7991975497879438,
+      "num_tokens": 642209396.0,
+      "step": 3670
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 7340,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 2,
+  "save_steps": 500.0,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.0682440167605109e+19,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-3670/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3c7a1cd4a8128372dd99461593bd550670f3a575287a7871a29132265b5bdcab
+size 6072

checkpoint-7340/chat_template.jinja

@@ -0,0 +1,21 @@
+
+    {{- bos_token }}
+    {%- for message in messages %}
+        {%- if message['role'] == 'assistant' %}
+            {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>' + '
+' }}
+            {%- generation %}
+            {{- message['content'] + '<|eot_id|>' }}
+            {%- endgeneration %}
+            {{- '
+' }}
+        {%- else %}
+            {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>' + '
+' + message['content'] + '<|eot_id|>' + '
+' }}
+        {%- endif %}
+    {%- endfor %}
+    {%- if add_generation_prompt %}
+        {{- '<|start_header_id|>assistant<|end_header_id|>
+' }}
+    {%- endif %}

checkpoint-7340/config.json

@@ -0,0 +1,130 @@
+{
+  "architectures": [
+    "HymbaForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "attn_hidden_size": -1,
+  "attn_implementation": "flash_attention_2",
+  "attn_implementation_new": "flash_attention_2",
+  "attn_only_wo_proj": true,
+  "auto_map": {
+    "AutoConfig": "configuration_hymba.HymbaConfig",
+    "AutoModelForCausalLM": "modeling_hymba.HymbaForCausalLM"
+  },
+  "bos_token_id": 128000,
+  "calc_logits_for_entire_prompt": false,
+  "conv_dim": {
+    "0": 3200,
+    "1": 3200,
+    "10": 3200,
+    "11": 3200,
+    "12": 3200,
+    "13": 3200,
+    "14": 3200,
+    "15": 3200,
+    "16": 3200,
+    "17": 3200,
+    "18": 3200,
+    "19": 3200,
+    "2": 3200,
+    "20": 3200,
+    "21": 3200,
+    "22": 3200,
+    "23": 3200,
+    "24": 3200,
+    "25": 3200,
+    "26": 3200,
+    "27": 3200,
+    "28": 3200,
+    "29": 3200,
+    "3": 3200,
+    "30": 3200,
+    "31": 3200,
+    "4": 3200,
+    "5": 3200,
+    "6": 3200,
+    "7": 3200,
+    "8": 3200,
+    "9": 3200
+  },
+  "eos_token_id": [
+    128001,
+    128008,
+    128009
+  ],
+  "global_attn_idx": [
+    0,
+    8,
+    15
+  ],
+  "hidden_act": "silu",
+  "hidden_size": 2048,
+  "initializer_range": 0.02,
+  "intermediate_size": 8192,
+  "kq_head_dim": 64,
+  "kq_norm": "none",
+  "kv_reuse_every_i_layer": -1,
+  "kv_reuse_group": null,
+  "kv_weight_reuse": false,
+  "layer_type": [
+    "h",
+    "h",
+    "h",
+    "h",
+    "h",
+    "h",
+    "h",
+    "h",
+    "h",
+    "h",
+    "h",
+    "h",
+    "h",
+    "h",
+    "h",
+    "h"
+  ],
+  "mamba_conv_bias": true,
+  "mamba_d_conv": 4,
+  "mamba_d_state": 16,
+  "mamba_dt_rank": 100,
+  "mamba_expand": 1,
+  "mamba_inner_layernorms": true,
+  "mamba_proj_bias": false,
+  "max_position_embeddings": 131072,
+  "memory_tokens_interspersed_every": 0,
+  "mlp_hidden_act": "silu",
+  "model_type": "hymba",
+  "num_attention_heads": 32,
+  "num_experts": 1,
+  "num_experts_per_tok": 1,
+  "num_hidden_layers": 16,
+  "num_key_value_heads": 8,
+  "num_mamba": 1,
+  "num_memory_tokens": 0,
+  "orig_max_position_embeddings": 2048,
+  "output_router_logits": false,
+  "pad_token_id": null,
+  "pure_attn": true,
+  "rms_norm_eps": 1e-05,
+  "rope": true,
+  "rope_scaling": {
+    "factor": 32.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_type": "llama3"
+  },
+  "rope_theta": 500000.0,
+  "rope_type": "ntk",
+  "router_aux_loss_coef": 0.001,
+  "seq_length": 8192,
+  "sliding_window": 1024,
+  "tie_word_embeddings": true,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.53.0",
+  "use_cache": false,
+  "use_mamba_kernels": true,
+  "v_head_dim": 64,
+  "vocab_size": 128256
+}

checkpoint-7340/configuration_hymba.py

@@ -0,0 +1,120 @@
+import math
+from transformers.configuration_utils import PretrainedConfig
+
+
+class HymbaConfig(PretrainedConfig):
+
+    model_type = "hymba"
+    keys_to_ignore_at_inference = ["past_key_values"]
+
+    def __init__(
+            self,
+            vocab_size=65536,
+            tie_word_embeddings=False,
+            hidden_size=4096,
+            intermediate_size=14336,
+            num_hidden_layers=32,
+            num_attention_heads=32,
+            num_key_value_heads=8,
+            hidden_act="silu",
+            initializer_range=0.02,
+            rms_norm_eps=1e-6,
+            attn_only_wo_proj=False,
+            use_cache=True,
+            calc_logits_for_entire_prompt=False,
+            output_router_logits=False,
+            router_aux_loss_coef=0.001,
+            pad_token_id=0,
+            bos_token_id=1,
+            eos_token_id=2,
+            sliding_window=None,
+            max_position_embeddings=262144,
+            orig_max_position_embeddings=None,
+            attention_dropout=0.0,
+            num_experts_per_tok=2,
+            num_experts=16,
+            use_mamba_kernels=True,
+            mamba_d_state=16,
+            mamba_d_conv=4,
+            mamba_expand=2,
+            mamba_dt_rank="auto",
+            mamba_conv_bias=True,
+            mamba_proj_bias=False,
+            mamba_inner_layernorms=True,
+            kv_reuse_every_i_layer=-1,
+            kv_reuse_group=None,
+            kv_weight_reuse=False,
+            global_attn_idx=None,
+            num_mamba=1,
+            pure_attn=False,
+            attn_implementation_new='sdpa',
+            rope_type=None,
+            **kwargs,
+    ):
+        self.vocab_size = vocab_size
+        self.tie_word_embeddings = tie_word_embeddings
+        self.hidden_size = hidden_size
+        self.intermediate_size = intermediate_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+        self.sliding_window = sliding_window
+        self.max_position_embeddings = max_position_embeddings
+        self.orig_max_position_embeddings = orig_max_position_embeddings
+        self.attention_dropout = attention_dropout
+        self.attn_only_wo_proj = attn_only_wo_proj
+
+        if num_key_value_heads is None:
+            num_key_value_heads = num_attention_heads
+
+        self.num_key_value_heads = num_key_value_heads
+        self.hidden_act = hidden_act
+        self.initializer_range = initializer_range
+        self.rms_norm_eps = rms_norm_eps
+
+        self.use_cache = use_cache
+        self.calc_logits_for_entire_prompt = calc_logits_for_entire_prompt
+        self.output_router_logits = output_router_logits
+        self.router_aux_loss_coef = router_aux_loss_coef
+
+        self.num_experts_per_tok = num_experts_per_tok
+        self.num_experts = num_experts
+
+        self.use_mamba_kernels = use_mamba_kernels
+        self.mamba_d_state = mamba_d_state
+        self.mamba_d_conv = mamba_d_conv
+        self.mamba_expand = mamba_expand
+        self.mamba_dt_rank = math.ceil(self.hidden_size / 16) if mamba_dt_rank == "auto" else mamba_dt_rank
+        self.mamba_conv_bias = mamba_conv_bias
+        self.mamba_proj_bias = mamba_proj_bias
+        self.mamba_inner_layernorms = mamba_inner_layernorms
+
+        self.attn_hidden_size = kwargs.pop("attn_hidden_size", -1)
+        self.kq_head_dim = kwargs.pop("kq_head_dim", -1)
+        self.v_head_dim = kwargs.pop("v_head_dim", -1)
+        self.kq_norm = kwargs.pop("kq_norm", None)
+        self.rope = kwargs.pop("rope", False)
+        self.rope_theta = kwargs.pop("rope_theta", 10000.0)
+        self.num_memory_tokens = kwargs.pop("num_memory_tokens", 0)
+        self.memory_tokens_interspersed_every = kwargs.pop("memory_tokens_interspersed_every", 0)
+
+        self.kv_reuse_every_i_layer = kv_reuse_every_i_layer
+        self.kv_reuse_group = kv_reuse_group
+        self.kv_weight_reuse = kv_weight_reuse
+
+        self.global_attn_idx = global_attn_idx
+
+        self.num_mamba = num_mamba
+
+        self.attn_implementation_new = attn_implementation_new
+
+        self.rope_type = rope_type
+
+        self.pure_attn = pure_attn
+        
+        super().__init__(
+            pad_token_id=pad_token_id,
+            bos_token_id=bos_token_id,
+            eos_token_id=eos_token_id,
+            tie_word_embeddings=tie_word_embeddings,
+            **kwargs,
+        )

checkpoint-7340/generation_config.json

@@ -0,0 +1,10 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 128000,
+  "eos_token_id": [
+    128001,
+    128008,
+    128009
+  ],
+  "transformers_version": "4.53.0"
+}

checkpoint-7340/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b784cc3cfebf62972a86fc158e0c19334775311da5158c2343b54ffb5098352c
+size 2471777104

checkpoint-7340/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:618b975659d8dfd5d04475b4268cdaa9ee406341cba5733c8a17dcbd6fad1202
+size 4943354486

checkpoint-7340/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1ff264f99d31b522cc7e2a4eac9d38606d0c58a34c0adc74d71e0ca8b371dc36
+size 14244

checkpoint-7340/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:594ec3f35b27867a9e9ee543fd5ef4137eea6d390cf479a3bd2b0ddd71f40896
+size 1064

checkpoint-7340/special_tokens_map.json

@@ -0,0 +1,17 @@
+{
+  "bos_token": {
+    "content": "<|begin_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|eot_id|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<|eot_id|>"
+}

checkpoint-7340/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920

checkpoint-7340/tokenizer_config.json

@@ -0,0 +1,2063 @@
+{
+  "added_tokens_decoder": {
+    "128000": {
+      "content": "<|begin_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128001": {
+      "content": "<|end_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128002": {
+      "content": "<|reserved_special_token_0|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128003": {
+      "content": "<|reserved_special_token_1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128004": {
+      "content": "<|finetune_right_pad_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128005": {
+      "content": "<|reserved_special_token_2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128006": {
+      "content": "<|start_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128007": {
+      "content": "<|end_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128008": {
+      "content": "<|eom_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128009": {
+      "content": "<|eot_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128010": {
+      "content": "<|python_tag|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128011": {
+      "content": "<|reserved_special_token_3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128012": {
+      "content": "<|reserved_special_token_4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128013": {
+      "content": "<|reserved_special_token_5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128014": {
+      "content": "<|reserved_special_token_6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128015": {
+      "content": "<|reserved_special_token_7|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128016": {
+      "content": "<|reserved_special_token_8|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128017": {
+      "content": "<|reserved_special_token_9|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128018": {
+      "content": "<|reserved_special_token_10|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128019": {
+      "content": "<|reserved_special_token_11|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128020": {
+      "content": "<|reserved_special_token_12|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128021": {
+      "content": "<|reserved_special_token_13|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128022": {
+      "content": "<|reserved_special_token_14|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128023": {
+      "content": "<|reserved_special_token_15|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128024": {
+      "content": "<|reserved_special_token_16|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128025": {
+      "content": "<|reserved_special_token_17|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128026": {
+      "content": "<|reserved_special_token_18|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128027": {
+      "content": "<|reserved_special_token_19|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128028": {
+      "content": "<|reserved_special_token_20|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128029": {
+      "content": "<|reserved_special_token_21|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128030": {
+      "content": "<|reserved_special_token_22|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128031": {
+      "content": "<|reserved_special_token_23|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128032": {
+      "content": "<|reserved_special_token_24|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128033": {
+      "content": "<|reserved_special_token_25|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128034": {
+      "content": "<|reserved_special_token_26|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128035": {
+      "content": "<|reserved_special_token_27|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128036": {
+      "content": "<|reserved_special_token_28|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128037": {
+      "content": "<|reserved_special_token_29|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128038": {
+      "content": "<|reserved_special_token_30|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128039": {
+      "content": "<|reserved_special_token_31|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128040": {
+      "content": "<|reserved_special_token_32|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128041": {
+      "content": "<|reserved_special_token_33|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128042": {
+      "content": "<|reserved_special_token_34|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128043": {
+      "content": "<|reserved_special_token_35|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128044": {
+      "content": "<|reserved_special_token_36|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128045": {
+      "content": "<|reserved_special_token_37|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128046": {
+      "content": "<|reserved_special_token_38|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128047": {
+      "content": "<|reserved_special_token_39|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128048": {
+      "content": "<|reserved_special_token_40|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128049": {
+      "content": "<|reserved_special_token_41|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128050": {
+      "content": "<|reserved_special_token_42|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128051": {
+      "content": "<|reserved_special_token_43|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128052": {
+      "content": "<|reserved_special_token_44|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128053": {
+      "content": "<|reserved_special_token_45|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128054": {
+      "content": "<|reserved_special_token_46|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128055": {
+      "content": "<|reserved_special_token_47|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128056": {
+      "content": "<|reserved_special_token_48|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128057": {
+      "content": "<|reserved_special_token_49|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128058": {
+      "content": "<|reserved_special_token_50|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128059": {
+      "content": "<|reserved_special_token_51|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128060": {
+      "content": "<|reserved_special_token_52|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128061": {
+      "content": "<|reserved_special_token_53|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128062": {
+      "content": "<|reserved_special_token_54|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128063": {
+      "content": "<|reserved_special_token_55|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128064": {
+      "content": "<|reserved_special_token_56|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128065": {
+      "content": "<|reserved_special_token_57|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128066": {
+      "content": "<|reserved_special_token_58|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128067": {
+      "content": "<|reserved_special_token_59|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128068": {
+      "content": "<|reserved_special_token_60|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128069": {
+      "content": "<|reserved_special_token_61|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128070": {
+      "content": "<|reserved_special_token_62|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128071": {
+      "content": "<|reserved_special_token_63|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128072": {
+      "content": "<|reserved_special_token_64|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128073": {
+      "content": "<|reserved_special_token_65|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128074": {
+      "content": "<|reserved_special_token_66|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128075": {
+      "content": "<|reserved_special_token_67|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128076": {
+      "content": "<|reserved_special_token_68|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128077": {
+      "content": "<|reserved_special_token_69|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128078": {
+      "content": "<|reserved_special_token_70|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128079": {
+      "content": "<|reserved_special_token_71|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128080": {
+      "content": "<|reserved_special_token_72|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128081": {
+      "content": "<|reserved_special_token_73|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128082": {
+      "content": "<|reserved_special_token_74|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128083": {
+      "content": "<|reserved_special_token_75|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128084": {
+      "content": "<|reserved_special_token_76|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128085": {
+      "content": "<|reserved_special_token_77|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128086": {
+      "content": "<|reserved_special_token_78|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128087": {
+      "content": "<|reserved_special_token_79|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128088": {
+      "content": "<|reserved_special_token_80|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128089": {
+      "content": "<|reserved_special_token_81|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128090": {
+      "content": "<|reserved_special_token_82|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128091": {
+      "content": "<|reserved_special_token_83|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128092": {
+      "content": "<|reserved_special_token_84|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128093": {
+      "content": "<|reserved_special_token_85|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128094": {
+      "content": "<|reserved_special_token_86|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128095": {
+      "content": "<|reserved_special_token_87|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128096": {
+      "content": "<|reserved_special_token_88|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128097": {
+      "content": "<|reserved_special_token_89|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128098": {
+      "content": "<|reserved_special_token_90|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128099": {
+      "content": "<|reserved_special_token_91|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128100": {
+      "content": "<|reserved_special_token_92|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128101": {
+      "content": "<|reserved_special_token_93|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128102": {
+      "content": "<|reserved_special_token_94|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128103": {
+      "content": "<|reserved_special_token_95|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128104": {
+      "content": "<|reserved_special_token_96|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128105": {
+      "content": "<|reserved_special_token_97|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128106": {
+      "content": "<|reserved_special_token_98|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128107": {
+      "content": "<|reserved_special_token_99|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128108": {
+      "content": "<|reserved_special_token_100|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128109": {
+      "content": "<|reserved_special_token_101|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128110": {
+      "content": "<|reserved_special_token_102|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128111": {
+      "content": "<|reserved_special_token_103|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128112": {
+      "content": "<|reserved_special_token_104|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128113": {
+      "content": "<|reserved_special_token_105|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128114": {
+      "content": "<|reserved_special_token_106|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128115": {
+      "content": "<|reserved_special_token_107|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128116": {
+      "content": "<|reserved_special_token_108|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128117": {
+      "content": "<|reserved_special_token_109|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128118": {
+      "content": "<|reserved_special_token_110|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128119": {
+      "content": "<|reserved_special_token_111|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128120": {
+      "content": "<|reserved_special_token_112|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128121": {
+      "content": "<|reserved_special_token_113|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128122": {
+      "content": "<|reserved_special_token_114|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128123": {
+      "content": "<|reserved_special_token_115|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128124": {
+      "content": "<|reserved_special_token_116|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128125": {
+      "content": "<|reserved_special_token_117|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128126": {
+      "content": "<|reserved_special_token_118|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128127": {
+      "content": "<|reserved_special_token_119|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128128": {
+      "content": "<|reserved_special_token_120|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128129": {
+      "content": "<|reserved_special_token_121|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128130": {
+      "content": "<|reserved_special_token_122|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128131": {
+      "content": "<|reserved_special_token_123|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128132": {
+      "content": "<|reserved_special_token_124|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128133": {
+      "content": "<|reserved_special_token_125|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128134": {
+      "content": "<|reserved_special_token_126|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128135": {
+      "content": "<|reserved_special_token_127|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128136": {
+      "content": "<|reserved_special_token_128|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128137": {
+      "content": "<|reserved_special_token_129|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128138": {
+      "content": "<|reserved_special_token_130|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128139": {
+      "content": "<|reserved_special_token_131|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128140": {
+      "content": "<|reserved_special_token_132|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128141": {
+      "content": "<|reserved_special_token_133|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128142": {
+      "content": "<|reserved_special_token_134|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128143": {
+      "content": "<|reserved_special_token_135|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128144": {
+      "content": "<|reserved_special_token_136|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128145": {
+      "content": "<|reserved_special_token_137|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128146": {
+      "content": "<|reserved_special_token_138|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128147": {
+      "content": "<|reserved_special_token_139|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128148": {
+      "content": "<|reserved_special_token_140|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128149": {
+      "content": "<|reserved_special_token_141|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128150": {
+      "content": "<|reserved_special_token_142|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128151": {
+      "content": "<|reserved_special_token_143|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128152": {
+      "content": "<|reserved_special_token_144|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128153": {
+      "content": "<|reserved_special_token_145|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128154": {
+      "content": "<|reserved_special_token_146|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128155": {
+      "content": "<|reserved_special_token_147|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128156": {
+      "content": "<|reserved_special_token_148|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128157": {
+      "content": "<|reserved_special_token_149|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128158": {
+      "content": "<|reserved_special_token_150|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128159": {
+      "content": "<|reserved_special_token_151|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128160": {
+      "content": "<|reserved_special_token_152|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128161": {
+      "content": "<|reserved_special_token_153|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128162": {
+      "content": "<|reserved_special_token_154|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128163": {
+      "content": "<|reserved_special_token_155|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128164": {
+      "content": "<|reserved_special_token_156|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128165": {
+      "content": "<|reserved_special_token_157|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128166": {
+      "content": "<|reserved_special_token_158|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128167": {
+      "content": "<|reserved_special_token_159|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128168": {
+      "content": "<|reserved_special_token_160|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128169": {
+      "content": "<|reserved_special_token_161|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128170": {
+      "content": "<|reserved_special_token_162|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128171": {
+      "content": "<|reserved_special_token_163|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128172": {
+      "content": "<|reserved_special_token_164|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128173": {
+      "content": "<|reserved_special_token_165|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128174": {
+      "content": "<|reserved_special_token_166|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128175": {
+      "content": "<|reserved_special_token_167|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128176": {
+      "content": "<|reserved_special_token_168|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128177": {
+      "content": "<|reserved_special_token_169|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128178": {
+      "content": "<|reserved_special_token_170|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128179": {
+      "content": "<|reserved_special_token_171|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128180": {
+      "content": "<|reserved_special_token_172|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128181": {
+      "content": "<|reserved_special_token_173|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128182": {
+      "content": "<|reserved_special_token_174|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128183": {
+      "content": "<|reserved_special_token_175|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128184": {
+      "content": "<|reserved_special_token_176|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128185": {
+      "content": "<|reserved_special_token_177|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128186": {
+      "content": "<|reserved_special_token_178|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128187": {
+      "content": "<|reserved_special_token_179|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128188": {
+      "content": "<|reserved_special_token_180|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128189": {
+      "content": "<|reserved_special_token_181|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128190": {
+      "content": "<|reserved_special_token_182|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128191": {
+      "content": "<|reserved_special_token_183|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128192": {
+      "content": "<|reserved_special_token_184|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128193": {
+      "content": "<|reserved_special_token_185|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128194": {
+      "content": "<|reserved_special_token_186|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128195": {
+      "content": "<|reserved_special_token_187|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128196": {
+      "content": "<|reserved_special_token_188|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128197": {
+      "content": "<|reserved_special_token_189|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128198": {
+      "content": "<|reserved_special_token_190|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128199": {
+      "content": "<|reserved_special_token_191|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128200": {
+      "content": "<|reserved_special_token_192|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128201": {
+      "content": "<|reserved_special_token_193|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128202": {
+      "content": "<|reserved_special_token_194|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128203": {
+      "content": "<|reserved_special_token_195|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128204": {
+      "content": "<|reserved_special_token_196|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128205": {
+      "content": "<|reserved_special_token_197|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128206": {
+      "content": "<|reserved_special_token_198|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128207": {
+      "content": "<|reserved_special_token_199|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128208": {
+      "content": "<|reserved_special_token_200|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128209": {
+      "content": "<|reserved_special_token_201|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128210": {
+      "content": "<|reserved_special_token_202|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128211": {
+      "content": "<|reserved_special_token_203|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128212": {
+      "content": "<|reserved_special_token_204|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128213": {
+      "content": "<|reserved_special_token_205|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128214": {
+      "content": "<|reserved_special_token_206|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128215": {
+      "content": "<|reserved_special_token_207|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128216": {
+      "content": "<|reserved_special_token_208|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128217": {
+      "content": "<|reserved_special_token_209|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128218": {
+      "content": "<|reserved_special_token_210|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128219": {
+      "content": "<|reserved_special_token_211|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128220": {
+      "content": "<|reserved_special_token_212|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128221": {
+      "content": "<|reserved_special_token_213|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128222": {
+      "content": "<|reserved_special_token_214|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128223": {
+      "content": "<|reserved_special_token_215|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128224": {
+      "content": "<|reserved_special_token_216|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128225": {
+      "content": "<|reserved_special_token_217|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128226": {
+      "content": "<|reserved_special_token_218|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128227": {
+      "content": "<|reserved_special_token_219|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128228": {
+      "content": "<|reserved_special_token_220|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128229": {
+      "content": "<|reserved_special_token_221|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128230": {
+      "content": "<|reserved_special_token_222|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128231": {
+      "content": "<|reserved_special_token_223|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128232": {
+      "content": "<|reserved_special_token_224|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128233": {
+      "content": "<|reserved_special_token_225|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128234": {
+      "content": "<|reserved_special_token_226|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128235": {
+      "content": "<|reserved_special_token_227|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128236": {
+      "content": "<|reserved_special_token_228|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128237": {
+      "content": "<|reserved_special_token_229|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128238": {
+      "content": "<|reserved_special_token_230|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128239": {
+      "content": "<|reserved_special_token_231|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128240": {
+      "content": "<|reserved_special_token_232|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128241": {
+      "content": "<|reserved_special_token_233|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128242": {
+      "content": "<|reserved_special_token_234|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128243": {
+      "content": "<|reserved_special_token_235|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128244": {
+      "content": "<|reserved_special_token_236|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128245": {
+      "content": "<|reserved_special_token_237|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128246": {
+      "content": "<|reserved_special_token_238|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128247": {
+      "content": "<|reserved_special_token_239|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128248": {
+      "content": "<|reserved_special_token_240|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128249": {
+      "content": "<|reserved_special_token_241|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128250": {
+      "content": "<|reserved_special_token_242|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128251": {
+      "content": "<|reserved_special_token_243|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128252": {
+      "content": "<|reserved_special_token_244|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128253": {
+      "content": "<|reserved_special_token_245|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128254": {
+      "content": "<|reserved_special_token_246|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128255": {
+      "content": "<|reserved_special_token_247|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|begin_of_text|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|eot_id|>",
+  "extra_special_tokens": {},
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "pad_token": "<|eot_id|>",
+  "tokenizer_class": "PreTrainedTokenizerFast"
+}

checkpoint-7340/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3c7a1cd4a8128372dd99461593bd550670f3a575287a7871a29132265b5bdcab
+size 6072

config.json

@@ -0,0 +1,130 @@
+{
+  "architectures": [
+    "HymbaForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "attn_hidden_size": -1,
+  "attn_implementation": "flash_attention_2",
+  "attn_implementation_new": "flash_attention_2",
+  "attn_only_wo_proj": true,
+  "auto_map": {
+    "AutoConfig": "configuration_hymba.HymbaConfig",
+    "AutoModelForCausalLM": "modeling_hymba.HymbaForCausalLM"
+  },
+  "bos_token_id": 128000,
+  "calc_logits_for_entire_prompt": false,
+  "conv_dim": {
+    "0": 3200,
+    "1": 3200,
+    "10": 3200,
+    "11": 3200,
+    "12": 3200,
+    "13": 3200,
+    "14": 3200,
+    "15": 3200,
+    "16": 3200,
+    "17": 3200,
+    "18": 3200,
+    "19": 3200,
+    "2": 3200,
+    "20": 3200,
+    "21": 3200,
+    "22": 3200,
+    "23": 3200,
+    "24": 3200,
+    "25": 3200,
+    "26": 3200,
+    "27": 3200,
+    "28": 3200,
+    "29": 3200,
+    "3": 3200,
+    "30": 3200,
+    "31": 3200,
+    "4": 3200,
+    "5": 3200,
+    "6": 3200,
+    "7": 3200,
+    "8": 3200,
+    "9": 3200
+  },
+  "eos_token_id": [
+    128001,
+    128008,
+    128009
+  ],
+  "global_attn_idx": [
+    0,
+    8,
+    15
+  ],
+  "hidden_act": "silu",
+  "hidden_size": 2048,
+  "initializer_range": 0.02,
+  "intermediate_size": 8192,
+  "kq_head_dim": 64,
+  "kq_norm": "none",
+  "kv_reuse_every_i_layer": -1,
+  "kv_reuse_group": null,
+  "kv_weight_reuse": false,
+  "layer_type": [
+    "h",
+    "h",
+    "h",
+    "h",
+    "h",
+    "h",
+    "h",
+    "h",
+    "h",
+    "h",
+    "h",
+    "h",
+    "h",
+    "h",
+    "h",
+    "h"
+  ],
+  "mamba_conv_bias": true,
+  "mamba_d_conv": 4,
+  "mamba_d_state": 16,
+  "mamba_dt_rank": 100,
+  "mamba_expand": 1,
+  "mamba_inner_layernorms": true,
+  "mamba_proj_bias": false,
+  "max_position_embeddings": 131072,
+  "memory_tokens_interspersed_every": 0,
+  "mlp_hidden_act": "silu",
+  "model_type": "hymba",
+  "num_attention_heads": 32,
+  "num_experts": 1,
+  "num_experts_per_tok": 1,
+  "num_hidden_layers": 16,
+  "num_key_value_heads": 8,
+  "num_mamba": 1,
+  "num_memory_tokens": 0,
+  "orig_max_position_embeddings": 2048,
+  "output_router_logits": false,
+  "pad_token_id": null,
+  "pure_attn": true,
+  "rms_norm_eps": 1e-05,
+  "rope": true,
+  "rope_scaling": {
+    "factor": 32.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_type": "llama3"
+  },
+  "rope_theta": 500000.0,
+  "rope_type": "ntk",
+  "router_aux_loss_coef": 0.001,
+  "seq_length": 8192,
+  "sliding_window": 1024,
+  "tie_word_embeddings": true,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.53.0",
+  "use_cache": false,
+  "use_mamba_kernels": true,
+  "v_head_dim": 64,
+  "vocab_size": 128256
+}

configuration_hymba.py

@@ -0,0 +1,120 @@
+import math
+from transformers.configuration_utils import PretrainedConfig
+
+
+class HymbaConfig(PretrainedConfig):
+
+    model_type = "hymba"
+    keys_to_ignore_at_inference = ["past_key_values"]
+
+    def __init__(
+            self,
+            vocab_size=65536,
+            tie_word_embeddings=False,
+            hidden_size=4096,
+            intermediate_size=14336,
+            num_hidden_layers=32,
+            num_attention_heads=32,
+            num_key_value_heads=8,
+            hidden_act="silu",
+            initializer_range=0.02,
+            rms_norm_eps=1e-6,
+            attn_only_wo_proj=False,
+            use_cache=True,
+            calc_logits_for_entire_prompt=False,
+            output_router_logits=False,
+            router_aux_loss_coef=0.001,
+            pad_token_id=0,
+            bos_token_id=1,
+            eos_token_id=2,
+            sliding_window=None,
+            max_position_embeddings=262144,
+            orig_max_position_embeddings=None,
+            attention_dropout=0.0,
+            num_experts_per_tok=2,
+            num_experts=16,
+            use_mamba_kernels=True,
+            mamba_d_state=16,
+            mamba_d_conv=4,
+            mamba_expand=2,
+            mamba_dt_rank="auto",
+            mamba_conv_bias=True,
+            mamba_proj_bias=False,
+            mamba_inner_layernorms=True,
+            kv_reuse_every_i_layer=-1,
+            kv_reuse_group=None,
+            kv_weight_reuse=False,
+            global_attn_idx=None,
+            num_mamba=1,
+            pure_attn=False,
+            attn_implementation_new='sdpa',
+            rope_type=None,
+            **kwargs,
+    ):
+        self.vocab_size = vocab_size
+        self.tie_word_embeddings = tie_word_embeddings
+        self.hidden_size = hidden_size
+        self.intermediate_size = intermediate_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+        self.sliding_window = sliding_window
+        self.max_position_embeddings = max_position_embeddings
+        self.orig_max_position_embeddings = orig_max_position_embeddings
+        self.attention_dropout = attention_dropout
+        self.attn_only_wo_proj = attn_only_wo_proj
+
+        if num_key_value_heads is None:
+            num_key_value_heads = num_attention_heads
+
+        self.num_key_value_heads = num_key_value_heads
+        self.hidden_act = hidden_act
+        self.initializer_range = initializer_range
+        self.rms_norm_eps = rms_norm_eps
+
+        self.use_cache = use_cache
+        self.calc_logits_for_entire_prompt = calc_logits_for_entire_prompt
+        self.output_router_logits = output_router_logits
+        self.router_aux_loss_coef = router_aux_loss_coef
+
+        self.num_experts_per_tok = num_experts_per_tok
+        self.num_experts = num_experts
+
+        self.use_mamba_kernels = use_mamba_kernels
+        self.mamba_d_state = mamba_d_state
+        self.mamba_d_conv = mamba_d_conv
+        self.mamba_expand = mamba_expand
+        self.mamba_dt_rank = math.ceil(self.hidden_size / 16) if mamba_dt_rank == "auto" else mamba_dt_rank
+        self.mamba_conv_bias = mamba_conv_bias
+        self.mamba_proj_bias = mamba_proj_bias
+        self.mamba_inner_layernorms = mamba_inner_layernorms
+
+        self.attn_hidden_size = kwargs.pop("attn_hidden_size", -1)
+        self.kq_head_dim = kwargs.pop("kq_head_dim", -1)
+        self.v_head_dim = kwargs.pop("v_head_dim", -1)
+        self.kq_norm = kwargs.pop("kq_norm", None)
+        self.rope = kwargs.pop("rope", False)
+        self.rope_theta = kwargs.pop("rope_theta", 10000.0)
+        self.num_memory_tokens = kwargs.pop("num_memory_tokens", 0)
+        self.memory_tokens_interspersed_every = kwargs.pop("memory_tokens_interspersed_every", 0)
+
+        self.kv_reuse_every_i_layer = kv_reuse_every_i_layer
+        self.kv_reuse_group = kv_reuse_group
+        self.kv_weight_reuse = kv_weight_reuse
+
+        self.global_attn_idx = global_attn_idx
+
+        self.num_mamba = num_mamba
+
+        self.attn_implementation_new = attn_implementation_new
+
+        self.rope_type = rope_type
+
+        self.pure_attn = pure_attn
+        
+        super().__init__(
+            pad_token_id=pad_token_id,
+            bos_token_id=bos_token_id,
+            eos_token_id=eos_token_id,
+            tie_word_embeddings=tie_word_embeddings,
+            **kwargs,
+        )

generation_config.json

@@ -0,0 +1,10 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 128000,
+  "eos_token_id": [
+    128001,
+    128008,
+    128009
+  ],
+  "transformers_version": "4.53.0"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b784cc3cfebf62972a86fc158e0c19334775311da5158c2343b54ffb5098352c
+size 2471777104

special_tokens_map.json

@@ -0,0 +1,17 @@
+{
+  "bos_token": {
+    "content": "<|begin_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|eot_id|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<|eot_id|>"
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920

tokenizer_config.json

@@ -0,0 +1,2063 @@
+{
+  "added_tokens_decoder": {
+    "128000": {
+      "content": "<|begin_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128001": {
+      "content": "<|end_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128002": {
+      "content": "<|reserved_special_token_0|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128003": {
+      "content": "<|reserved_special_token_1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128004": {
+      "content": "<|finetune_right_pad_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128005": {
+      "content": "<|reserved_special_token_2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128006": {
+      "content": "<|start_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128007": {
+      "content": "<|end_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128008": {
+      "content": "<|eom_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128009": {
+      "content": "<|eot_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128010": {
+      "content": "<|python_tag|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128011": {
+      "content": "<|reserved_special_token_3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128012": {
+      "content": "<|reserved_special_token_4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128013": {
+      "content": "<|reserved_special_token_5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128014": {
+      "content": "<|reserved_special_token_6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128015": {
+      "content": "<|reserved_special_token_7|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128016": {
+      "content": "<|reserved_special_token_8|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128017": {
+      "content": "<|reserved_special_token_9|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128018": {
+      "content": "<|reserved_special_token_10|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128019": {
+      "content": "<|reserved_special_token_11|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128020": {
+      "content": "<|reserved_special_token_12|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128021": {
+      "content": "<|reserved_special_token_13|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128022": {
+      "content": "<|reserved_special_token_14|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128023": {
+      "content": "<|reserved_special_token_15|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128024": {
+      "content": "<|reserved_special_token_16|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128025": {
+      "content": "<|reserved_special_token_17|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128026": {
+      "content": "<|reserved_special_token_18|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128027": {
+      "content": "<|reserved_special_token_19|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128028": {
+      "content": "<|reserved_special_token_20|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128029": {
+      "content": "<|reserved_special_token_21|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128030": {
+      "content": "<|reserved_special_token_22|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128031": {
+      "content": "<|reserved_special_token_23|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128032": {
+      "content": "<|reserved_special_token_24|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128033": {
+      "content": "<|reserved_special_token_25|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128034": {
+      "content": "<|reserved_special_token_26|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128035": {
+      "content": "<|reserved_special_token_27|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128036": {
+      "content": "<|reserved_special_token_28|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128037": {
+      "content": "<|reserved_special_token_29|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128038": {
+      "content": "<|reserved_special_token_30|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128039": {
+      "content": "<|reserved_special_token_31|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128040": {
+      "content": "<|reserved_special_token_32|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128041": {
+      "content": "<|reserved_special_token_33|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128042": {
+      "content": "<|reserved_special_token_34|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128043": {
+      "content": "<|reserved_special_token_35|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128044": {
+      "content": "<|reserved_special_token_36|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128045": {
+      "content": "<|reserved_special_token_37|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128046": {
+      "content": "<|reserved_special_token_38|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128047": {
+      "content": "<|reserved_special_token_39|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128048": {
+      "content": "<|reserved_special_token_40|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128049": {
+      "content": "<|reserved_special_token_41|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128050": {
+      "content": "<|reserved_special_token_42|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128051": {
+      "content": "<|reserved_special_token_43|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128052": {
+      "content": "<|reserved_special_token_44|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128053": {
+      "content": "<|reserved_special_token_45|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128054": {
+      "content": "<|reserved_special_token_46|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128055": {
+      "content": "<|reserved_special_token_47|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128056": {
+      "content": "<|reserved_special_token_48|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128057": {
+      "content": "<|reserved_special_token_49|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128058": {
+      "content": "<|reserved_special_token_50|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128059": {
+      "content": "<|reserved_special_token_51|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128060": {
+      "content": "<|reserved_special_token_52|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128061": {
+      "content": "<|reserved_special_token_53|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128062": {
+      "content": "<|reserved_special_token_54|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128063": {
+      "content": "<|reserved_special_token_55|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128064": {
+      "content": "<|reserved_special_token_56|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128065": {
+      "content": "<|reserved_special_token_57|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128066": {
+      "content": "<|reserved_special_token_58|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128067": {
+      "content": "<|reserved_special_token_59|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128068": {
+      "content": "<|reserved_special_token_60|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128069": {
+      "content": "<|reserved_special_token_61|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128070": {
+      "content": "<|reserved_special_token_62|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128071": {
+      "content": "<|reserved_special_token_63|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128072": {
+      "content": "<|reserved_special_token_64|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128073": {
+      "content": "<|reserved_special_token_65|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128074": {
+      "content": "<|reserved_special_token_66|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128075": {
+      "content": "<|reserved_special_token_67|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128076": {
+      "content": "<|reserved_special_token_68|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128077": {
+      "content": "<|reserved_special_token_69|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128078": {
+      "content": "<|reserved_special_token_70|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128079": {
+      "content": "<|reserved_special_token_71|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128080": {
+      "content": "<|reserved_special_token_72|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128081": {
+      "content": "<|reserved_special_token_73|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128082": {
+      "content": "<|reserved_special_token_74|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128083": {
+      "content": "<|reserved_special_token_75|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128084": {
+      "content": "<|reserved_special_token_76|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128085": {
+      "content": "<|reserved_special_token_77|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128086": {
+      "content": "<|reserved_special_token_78|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128087": {
+      "content": "<|reserved_special_token_79|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128088": {
+      "content": "<|reserved_special_token_80|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128089": {
+      "content": "<|reserved_special_token_81|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128090": {
+      "content": "<|reserved_special_token_82|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128091": {
+      "content": "<|reserved_special_token_83|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128092": {
+      "content": "<|reserved_special_token_84|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128093": {
+      "content": "<|reserved_special_token_85|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128094": {
+      "content": "<|reserved_special_token_86|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128095": {
+      "content": "<|reserved_special_token_87|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128096": {
+      "content": "<|reserved_special_token_88|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128097": {
+      "content": "<|reserved_special_token_89|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128098": {
+      "content": "<|reserved_special_token_90|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128099": {
+      "content": "<|reserved_special_token_91|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128100": {
+      "content": "<|reserved_special_token_92|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128101": {
+      "content": "<|reserved_special_token_93|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128102": {
+      "content": "<|reserved_special_token_94|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128103": {
+      "content": "<|reserved_special_token_95|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128104": {
+      "content": "<|reserved_special_token_96|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128105": {
+      "content": "<|reserved_special_token_97|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128106": {
+      "content": "<|reserved_special_token_98|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128107": {
+      "content": "<|reserved_special_token_99|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128108": {
+      "content": "<|reserved_special_token_100|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128109": {
+      "content": "<|reserved_special_token_101|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128110": {
+      "content": "<|reserved_special_token_102|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128111": {
+      "content": "<|reserved_special_token_103|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128112": {
+      "content": "<|reserved_special_token_104|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128113": {
+      "content": "<|reserved_special_token_105|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128114": {
+      "content": "<|reserved_special_token_106|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128115": {
+      "content": "<|reserved_special_token_107|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128116": {
+      "content": "<|reserved_special_token_108|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128117": {
+      "content": "<|reserved_special_token_109|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128118": {
+      "content": "<|reserved_special_token_110|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128119": {
+      "content": "<|reserved_special_token_111|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128120": {
+      "content": "<|reserved_special_token_112|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128121": {
+      "content": "<|reserved_special_token_113|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128122": {
+      "content": "<|reserved_special_token_114|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128123": {
+      "content": "<|reserved_special_token_115|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128124": {
+      "content": "<|reserved_special_token_116|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128125": {
+      "content": "<|reserved_special_token_117|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128126": {
+      "content": "<|reserved_special_token_118|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128127": {
+      "content": "<|reserved_special_token_119|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128128": {
+      "content": "<|reserved_special_token_120|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128129": {
+      "content": "<|reserved_special_token_121|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128130": {
+      "content": "<|reserved_special_token_122|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128131": {
+      "content": "<|reserved_special_token_123|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128132": {
+      "content": "<|reserved_special_token_124|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128133": {
+      "content": "<|reserved_special_token_125|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128134": {
+      "content": "<|reserved_special_token_126|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128135": {
+      "content": "<|reserved_special_token_127|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128136": {
+      "content": "<|reserved_special_token_128|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128137": {
+      "content": "<|reserved_special_token_129|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128138": {
+      "content": "<|reserved_special_token_130|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128139": {
+      "content": "<|reserved_special_token_131|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128140": {
+      "content": "<|reserved_special_token_132|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128141": {
+      "content": "<|reserved_special_token_133|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128142": {
+      "content": "<|reserved_special_token_134|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128143": {
+      "content": "<|reserved_special_token_135|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128144": {
+      "content": "<|reserved_special_token_136|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128145": {
+      "content": "<|reserved_special_token_137|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128146": {
+      "content": "<|reserved_special_token_138|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128147": {
+      "content": "<|reserved_special_token_139|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128148": {
+      "content": "<|reserved_special_token_140|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128149": {
+      "content": "<|reserved_special_token_141|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128150": {
+      "content": "<|reserved_special_token_142|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128151": {
+      "content": "<|reserved_special_token_143|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128152": {
+      "content": "<|reserved_special_token_144|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128153": {
+      "content": "<|reserved_special_token_145|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128154": {
+      "content": "<|reserved_special_token_146|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128155": {
+      "content": "<|reserved_special_token_147|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128156": {
+      "content": "<|reserved_special_token_148|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128157": {
+      "content": "<|reserved_special_token_149|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128158": {
+      "content": "<|reserved_special_token_150|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128159": {
+      "content": "<|reserved_special_token_151|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128160": {
+      "content": "<|reserved_special_token_152|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128161": {
+      "content": "<|reserved_special_token_153|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128162": {
+      "content": "<|reserved_special_token_154|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128163": {
+      "content": "<|reserved_special_token_155|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128164": {
+      "content": "<|reserved_special_token_156|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128165": {
+      "content": "<|reserved_special_token_157|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128166": {
+      "content": "<|reserved_special_token_158|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128167": {
+      "content": "<|reserved_special_token_159|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128168": {
+      "content": "<|reserved_special_token_160|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128169": {
+      "content": "<|reserved_special_token_161|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128170": {
+      "content": "<|reserved_special_token_162|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128171": {
+      "content": "<|reserved_special_token_163|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128172": {
+      "content": "<|reserved_special_token_164|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128173": {
+      "content": "<|reserved_special_token_165|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128174": {
+      "content": "<|reserved_special_token_166|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128175": {
+      "content": "<|reserved_special_token_167|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128176": {
+      "content": "<|reserved_special_token_168|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128177": {
+      "content": "<|reserved_special_token_169|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128178": {
+      "content": "<|reserved_special_token_170|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128179": {
+      "content": "<|reserved_special_token_171|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128180": {
+      "content": "<|reserved_special_token_172|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128181": {
+      "content": "<|reserved_special_token_173|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128182": {
+      "content": "<|reserved_special_token_174|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128183": {
+      "content": "<|reserved_special_token_175|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128184": {
+      "content": "<|reserved_special_token_176|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128185": {
+      "content": "<|reserved_special_token_177|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128186": {
+      "content": "<|reserved_special_token_178|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128187": {
+      "content": "<|reserved_special_token_179|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128188": {
+      "content": "<|reserved_special_token_180|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128189": {
+      "content": "<|reserved_special_token_181|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128190": {
+      "content": "<|reserved_special_token_182|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128191": {
+      "content": "<|reserved_special_token_183|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128192": {
+      "content": "<|reserved_special_token_184|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128193": {
+      "content": "<|reserved_special_token_185|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128194": {
+      "content": "<|reserved_special_token_186|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128195": {
+      "content": "<|reserved_special_token_187|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128196": {
+      "content": "<|reserved_special_token_188|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128197": {
+      "content": "<|reserved_special_token_189|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128198": {
+      "content": "<|reserved_special_token_190|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128199": {
+      "content": "<|reserved_special_token_191|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128200": {
+      "content": "<|reserved_special_token_192|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128201": {
+      "content": "<|reserved_special_token_193|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128202": {
+      "content": "<|reserved_special_token_194|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128203": {
+      "content": "<|reserved_special_token_195|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128204": {
+      "content": "<|reserved_special_token_196|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128205": {
+      "content": "<|reserved_special_token_197|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128206": {
+      "content": "<|reserved_special_token_198|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128207": {
+      "content": "<|reserved_special_token_199|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128208": {
+      "content": "<|reserved_special_token_200|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128209": {
+      "content": "<|reserved_special_token_201|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128210": {
+      "content": "<|reserved_special_token_202|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128211": {
+      "content": "<|reserved_special_token_203|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128212": {
+      "content": "<|reserved_special_token_204|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128213": {
+      "content": "<|reserved_special_token_205|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128214": {
+      "content": "<|reserved_special_token_206|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128215": {
+      "content": "<|reserved_special_token_207|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128216": {
+      "content": "<|reserved_special_token_208|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128217": {
+      "content": "<|reserved_special_token_209|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128218": {
+      "content": "<|reserved_special_token_210|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128219": {
+      "content": "<|reserved_special_token_211|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128220": {
+      "content": "<|reserved_special_token_212|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128221": {
+      "content": "<|reserved_special_token_213|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128222": {
+      "content": "<|reserved_special_token_214|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128223": {
+      "content": "<|reserved_special_token_215|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128224": {
+      "content": "<|reserved_special_token_216|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128225": {
+      "content": "<|reserved_special_token_217|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128226": {
+      "content": "<|reserved_special_token_218|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128227": {
+      "content": "<|reserved_special_token_219|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128228": {
+      "content": "<|reserved_special_token_220|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128229": {
+      "content": "<|reserved_special_token_221|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128230": {
+      "content": "<|reserved_special_token_222|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128231": {
+      "content": "<|reserved_special_token_223|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128232": {
+      "content": "<|reserved_special_token_224|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128233": {
+      "content": "<|reserved_special_token_225|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128234": {
+      "content": "<|reserved_special_token_226|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128235": {
+      "content": "<|reserved_special_token_227|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128236": {
+      "content": "<|reserved_special_token_228|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128237": {
+      "content": "<|reserved_special_token_229|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128238": {
+      "content": "<|reserved_special_token_230|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128239": {
+      "content": "<|reserved_special_token_231|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128240": {
+      "content": "<|reserved_special_token_232|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128241": {
+      "content": "<|reserved_special_token_233|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128242": {
+      "content": "<|reserved_special_token_234|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128243": {
+      "content": "<|reserved_special_token_235|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128244": {
+      "content": "<|reserved_special_token_236|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128245": {
+      "content": "<|reserved_special_token_237|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128246": {
+      "content": "<|reserved_special_token_238|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128247": {
+      "content": "<|reserved_special_token_239|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128248": {
+      "content": "<|reserved_special_token_240|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128249": {
+      "content": "<|reserved_special_token_241|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128250": {
+      "content": "<|reserved_special_token_242|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128251": {
+      "content": "<|reserved_special_token_243|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128252": {
+      "content": "<|reserved_special_token_244|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128253": {
+      "content": "<|reserved_special_token_245|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128254": {
+      "content": "<|reserved_special_token_246|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128255": {
+      "content": "<|reserved_special_token_247|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|begin_of_text|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|eot_id|>",
+  "extra_special_tokens": {},
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "pad_token": "<|eot_id|>",
+  "tokenizer_class": "PreTrainedTokenizerFast"
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3c7a1cd4a8128372dd99461593bd550670f3a575287a7871a29132265b5bdcab
+size 6072