Upload folder using huggingface_hub

.gitattributes

@@ -57,3 +57,57 @@ barexam_qa_train_knowledge_100_instruct/5_128_e3_3e-5/tokenizer.json filter=lfs
 barexam_qa_train_knowledge_100_instruct/6_128_e3_3e-5/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 barexam_qa_train_knowledge_100_instruct/7_128_e3_3e-5/tokenizer.json filter=lfs diff=lfs merge=lfs -text
 barexam_qa_train_knowledge_100_instruct/8_128_e3_3e-5/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+hotpotqa_skill_paraphrase_4_base_3epoch/64_e1_3.0e-05/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+hotpotqa_skill_paraphrase_4_instruct_3epoch/64_e1_3.0e-05/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+hotpotqa_test_knowledge_50_base/0_128_e3_3e-5/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+hotpotqa_test_knowledge_50_base/1_128_e3_3e-5/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+hotpotqa_test_knowledge_50_base/2_128_e3_3e-5/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+hotpotqa_test_knowledge_50_base/3_128_e3_3e-5/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+hotpotqa_test_knowledge_50_base/4_128_e3_3e-5/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+hotpotqa_test_knowledge_50_base/5_128_e3_3e-5/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+hotpotqa_test_knowledge_50_instruct/0_128_e3_3e-5/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+hotpotqa_test_knowledge_50_instruct/1_128_e3_3e-5/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+hotpotqa_test_knowledge_50_instruct/2_128_e3_3e-5/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+hotpotqa_test_knowledge_50_instruct/3_128_e3_3e-5/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+hotpotqa_test_knowledge_50_instruct/4_128_e3_3e-5/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+hotpotqa_test_knowledge_50_instruct/5_128_e3_3e-5/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+hotpotqa_train_knowledge_50_base/0_128_e3_3e-5/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+hotpotqa_train_knowledge_50_base/10_128_e3_3e-5/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+hotpotqa_train_knowledge_50_base/11_128_e3_3e-5/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+hotpotqa_train_knowledge_50_base/12_128_e3_3e-5/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+hotpotqa_train_knowledge_50_base/13_128_e3_3e-5/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+hotpotqa_train_knowledge_50_base/14_128_e3_3e-5/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+hotpotqa_train_knowledge_50_base/15_128_e3_3e-5/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+hotpotqa_train_knowledge_50_base/16_128_e3_3e-5/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+hotpotqa_train_knowledge_50_base/17_128_e3_3e-5/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+hotpotqa_train_knowledge_50_base/18_128_e3_3e-5/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+hotpotqa_train_knowledge_50_base/19_128_e3_3e-5/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+hotpotqa_train_knowledge_50_base/1_128_e3_3e-5/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+hotpotqa_train_knowledge_50_base/2_128_e3_3e-5/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+hotpotqa_train_knowledge_50_base/3_128_e3_3e-5/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+hotpotqa_train_knowledge_50_base/4_128_e3_3e-5/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+hotpotqa_train_knowledge_50_base/5_128_e3_3e-5/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+hotpotqa_train_knowledge_50_base/6_128_e3_3e-5/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+hotpotqa_train_knowledge_50_base/7_128_e3_3e-5/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+hotpotqa_train_knowledge_50_base/8_128_e3_3e-5/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+hotpotqa_train_knowledge_50_base/9_128_e3_3e-5/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+hotpotqa_train_knowledge_50_instruct/0_128_e3_3e-5/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+hotpotqa_train_knowledge_50_instruct/10_128_e3_3e-5/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+hotpotqa_train_knowledge_50_instruct/11_128_e3_3e-5/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+hotpotqa_train_knowledge_50_instruct/12_128_e3_3e-5/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+hotpotqa_train_knowledge_50_instruct/13_128_e3_3e-5/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+hotpotqa_train_knowledge_50_instruct/14_128_e3_3e-5/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+hotpotqa_train_knowledge_50_instruct/15_128_e3_3e-5/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+hotpotqa_train_knowledge_50_instruct/16_128_e3_3e-5/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+hotpotqa_train_knowledge_50_instruct/17_128_e3_3e-5/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+hotpotqa_train_knowledge_50_instruct/18_128_e3_3e-5/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+hotpotqa_train_knowledge_50_instruct/19_128_e3_3e-5/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+hotpotqa_train_knowledge_50_instruct/1_128_e3_3e-5/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+hotpotqa_train_knowledge_50_instruct/2_128_e3_3e-5/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+hotpotqa_train_knowledge_50_instruct/3_128_e3_3e-5/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+hotpotqa_train_knowledge_50_instruct/4_128_e3_3e-5/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+hotpotqa_train_knowledge_50_instruct/5_128_e3_3e-5/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+hotpotqa_train_knowledge_50_instruct/6_128_e3_3e-5/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+hotpotqa_train_knowledge_50_instruct/7_128_e3_3e-5/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+hotpotqa_train_knowledge_50_instruct/8_128_e3_3e-5/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+hotpotqa_train_knowledge_50_instruct/9_128_e3_3e-5/tokenizer.json filter=lfs diff=lfs merge=lfs -text

hotpotqa_skill_paraphrase_4_base_3epoch/64_e1_3.0e-05/adapter_config.json

@@ -0,0 +1,39 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "meta-llama/Llama-3.1-8B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": false,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 128,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 64,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "v_proj",
+    "up_proj",
+    "k_proj",
+    "down_proj",
+    "q_proj",
+    "o_proj",
+    "gate_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_rslora": false
+}

hotpotqa_skill_paraphrase_4_base_3epoch/64_e1_3.0e-05/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7755f47750ba602d4c0d1bd4d6eab9c703219eafa33741dd03c79b6c0f9d51b9
+size 335605144

hotpotqa_skill_paraphrase_4_base_3epoch/64_e1_3.0e-05/all_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 1.0,
+    "total_flos": 3.9356831750946816e+17,
+    "train_loss": 3.96218955618414,
+    "train_runtime": 277.7455,
+    "train_samples": 2802,
+    "train_samples_per_second": 10.088,
+    "train_steps_per_second": 0.634
+}

hotpotqa_skill_paraphrase_4_base_3epoch/64_e1_3.0e-05/chat_template.jinja

@@ -0,0 +1,109 @@
+{{- bos_token }}
+{%- if custom_tools is defined %}
+    {%- set tools = custom_tools %}
+{%- endif %}
+{%- if not tools_in_user_message is defined %}
+    {%- set tools_in_user_message = true %}
+{%- endif %}
+{%- if not date_string is defined %}
+    {%- set date_string = "26 Jul 2024" %}
+{%- endif %}
+{%- if not tools is defined %}
+    {%- set tools = none %}
+{%- endif %}
+
+{#- This block extracts the system message, so we can slot it into the right place. #}
+{%- if messages[0]['role'] == 'system' %}
+    {%- set system_message = messages[0]['content']|trim %}
+    {%- set messages = messages[1:] %}
+{%- else %}
+    {%- set system_message = "" %}
+{%- endif %}
+
+{#- System message + builtin tools #}
+{{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
+{%- if builtin_tools is defined or tools is not none %}
+    {{- "Environment: ipython\n" }}
+{%- endif %}
+{%- if builtin_tools is defined %}
+    {{- "Tools: " + builtin_tools | reject('equalto', 'code_interpreter') | join(", ") + "\n\n"}}
+{%- endif %}
+{{- "Cutting Knowledge Date: December 2023\n" }}
+{{- "Today Date: " + date_string + "\n\n" }}
+{%- if tools is not none and not tools_in_user_message %}
+    {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }}
+    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+    {{- "Do not use variables.\n\n" }}
+    {%- for t in tools %}
+        {{- t | tojson(indent=4) }}
+        {{- "\n\n" }}
+    {%- endfor %}
+{%- endif %}
+{{- system_message }}
+{{- "<|eot_id|>" }}
+
+{#- Custom tools are passed in a user message with some extra guidance #}
+{%- if tools_in_user_message and not tools is none %}
+    {#- Extract the first user message so we can plug it in here #}
+    {%- if messages | length != 0 %}
+        {%- set first_user_message = messages[0]['content']|trim %}
+        {%- set messages = messages[1:] %}
+    {%- else %}
+        {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
+{%- endif %}
+    {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}}
+    {{- "Given the following functions, please respond with a JSON for a function call " }}
+    {{- "with its proper arguments that best answers the given prompt.\n\n" }}
+    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+    {{- "Do not use variables.\n\n" }}
+    {%- for t in tools %}
+        {{- t | tojson(indent=4) }}
+        {{- "\n\n" }}
+    {%- endfor %}
+    {{- first_user_message + "<|eot_id|>"}}
+{%- endif %}
+
+{%- for message in messages %}
+    {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
+        {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }}
+    {%- elif 'tool_calls' in message %}
+        {%- if not message.tool_calls|length == 1 %}
+            {{- raise_exception("This model only supports single tool-calls at once!") }}
+        {%- endif %}
+        {%- set tool_call = message.tool_calls[0].function %}
+        {%- if builtin_tools is defined and tool_call.name in builtin_tools %}
+            {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
+            {{- "<|python_tag|>" + tool_call.name + ".call(" }}
+            {%- for arg_name, arg_val in tool_call.arguments | items %}
+                {{- arg_name + '="' + arg_val + '"' }}
+                {%- if not loop.last %}
+                    {{- ", " }}
+                {%- endif %}
+                {%- endfor %}
+            {{- ")" }}
+        {%- else  %}
+            {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
+            {{- '{"name": "' + tool_call.name + '", ' }}
+            {{- '"parameters": ' }}
+            {{- tool_call.arguments | tojson }}
+            {{- "}" }}
+        {%- endif %}
+        {%- if builtin_tools is defined %}
+            {#- This means we're in ipython mode #}
+            {{- "<|eom_id|>" }}
+        {%- else %}
+            {{- "<|eot_id|>" }}
+        {%- endif %}
+    {%- elif message.role == "tool" or message.role == "ipython" %}
+        {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }}
+        {%- if message.content is mapping or message.content is iterable %}
+            {{- message.content | tojson }}
+        {%- else %}
+            {{- message.content }}
+        {%- endif %}
+        {{- "<|eot_id|>" }}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
+{%- endif %}

hotpotqa_skill_paraphrase_4_base_3epoch/64_e1_3.0e-05/config.json

@@ -0,0 +1,39 @@
+{
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "eos_token_id": [
+    128001,
+    128008,
+    128009
+  ],
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": {
+    "factor": 8.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_type": "llama3"
+  },
+  "rope_theta": 500000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.52.4",
+  "use_cache": true,
+  "vocab_size": 128256
+}

hotpotqa_skill_paraphrase_4_base_3epoch/64_e1_3.0e-05/generation_config.json

@@ -0,0 +1,12 @@
+{
+  "bos_token_id": 128000,
+  "do_sample": true,
+  "eos_token_id": [
+    128001,
+    128008,
+    128009
+  ],
+  "temperature": 0.6,
+  "top_p": 0.9,
+  "transformers_version": "4.52.4"
+}

hotpotqa_skill_paraphrase_4_base_3epoch/64_e1_3.0e-05/special_tokens_map.json

@@ -0,0 +1,17 @@
+{
+  "bos_token": {
+    "content": "<|begin_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|eot_id|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<|reserved_special_token_247|>"
+}

hotpotqa_skill_paraphrase_4_base_3epoch/64_e1_3.0e-05/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920

hotpotqa_skill_paraphrase_4_base_3epoch/64_e1_3.0e-05/tokenizer_config.json

@@ -0,0 +1,2063 @@
+{
+  "added_tokens_decoder": {
+    "128000": {
+      "content": "<|begin_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128001": {
+      "content": "<|end_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128002": {
+      "content": "<|reserved_special_token_0|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128003": {
+      "content": "<|reserved_special_token_1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128004": {
+      "content": "<|finetune_right_pad_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128005": {
+      "content": "<|reserved_special_token_2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128006": {
+      "content": "<|start_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128007": {
+      "content": "<|end_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128008": {
+      "content": "<|eom_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128009": {
+      "content": "<|eot_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128010": {
+      "content": "<|python_tag|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128011": {
+      "content": "<|reserved_special_token_3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128012": {
+      "content": "<|reserved_special_token_4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128013": {
+      "content": "<|reserved_special_token_5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128014": {
+      "content": "<|reserved_special_token_6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128015": {
+      "content": "<|reserved_special_token_7|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128016": {
+      "content": "<|reserved_special_token_8|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128017": {
+      "content": "<|reserved_special_token_9|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128018": {
+      "content": "<|reserved_special_token_10|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128019": {
+      "content": "<|reserved_special_token_11|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128020": {
+      "content": "<|reserved_special_token_12|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128021": {
+      "content": "<|reserved_special_token_13|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128022": {
+      "content": "<|reserved_special_token_14|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128023": {
+      "content": "<|reserved_special_token_15|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128024": {
+      "content": "<|reserved_special_token_16|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128025": {
+      "content": "<|reserved_special_token_17|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128026": {
+      "content": "<|reserved_special_token_18|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128027": {
+      "content": "<|reserved_special_token_19|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128028": {
+      "content": "<|reserved_special_token_20|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128029": {
+      "content": "<|reserved_special_token_21|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128030": {
+      "content": "<|reserved_special_token_22|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128031": {
+      "content": "<|reserved_special_token_23|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128032": {
+      "content": "<|reserved_special_token_24|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128033": {
+      "content": "<|reserved_special_token_25|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128034": {
+      "content": "<|reserved_special_token_26|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128035": {
+      "content": "<|reserved_special_token_27|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128036": {
+      "content": "<|reserved_special_token_28|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128037": {
+      "content": "<|reserved_special_token_29|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128038": {
+      "content": "<|reserved_special_token_30|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128039": {
+      "content": "<|reserved_special_token_31|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128040": {
+      "content": "<|reserved_special_token_32|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128041": {
+      "content": "<|reserved_special_token_33|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128042": {
+      "content": "<|reserved_special_token_34|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128043": {
+      "content": "<|reserved_special_token_35|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128044": {
+      "content": "<|reserved_special_token_36|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128045": {
+      "content": "<|reserved_special_token_37|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128046": {
+      "content": "<|reserved_special_token_38|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128047": {
+      "content": "<|reserved_special_token_39|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128048": {
+      "content": "<|reserved_special_token_40|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128049": {
+      "content": "<|reserved_special_token_41|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128050": {
+      "content": "<|reserved_special_token_42|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128051": {
+      "content": "<|reserved_special_token_43|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128052": {
+      "content": "<|reserved_special_token_44|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128053": {
+      "content": "<|reserved_special_token_45|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128054": {
+      "content": "<|reserved_special_token_46|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128055": {
+      "content": "<|reserved_special_token_47|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128056": {
+      "content": "<|reserved_special_token_48|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128057": {
+      "content": "<|reserved_special_token_49|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128058": {
+      "content": "<|reserved_special_token_50|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128059": {
+      "content": "<|reserved_special_token_51|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128060": {
+      "content": "<|reserved_special_token_52|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128061": {
+      "content": "<|reserved_special_token_53|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128062": {
+      "content": "<|reserved_special_token_54|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128063": {
+      "content": "<|reserved_special_token_55|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128064": {
+      "content": "<|reserved_special_token_56|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128065": {
+      "content": "<|reserved_special_token_57|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128066": {
+      "content": "<|reserved_special_token_58|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128067": {
+      "content": "<|reserved_special_token_59|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128068": {
+      "content": "<|reserved_special_token_60|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128069": {
+      "content": "<|reserved_special_token_61|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128070": {
+      "content": "<|reserved_special_token_62|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128071": {
+      "content": "<|reserved_special_token_63|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128072": {
+      "content": "<|reserved_special_token_64|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128073": {
+      "content": "<|reserved_special_token_65|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128074": {
+      "content": "<|reserved_special_token_66|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128075": {
+      "content": "<|reserved_special_token_67|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128076": {
+      "content": "<|reserved_special_token_68|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128077": {
+      "content": "<|reserved_special_token_69|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128078": {
+      "content": "<|reserved_special_token_70|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128079": {
+      "content": "<|reserved_special_token_71|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128080": {
+      "content": "<|reserved_special_token_72|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128081": {
+      "content": "<|reserved_special_token_73|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128082": {
+      "content": "<|reserved_special_token_74|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128083": {
+      "content": "<|reserved_special_token_75|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128084": {
+      "content": "<|reserved_special_token_76|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128085": {
+      "content": "<|reserved_special_token_77|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128086": {
+      "content": "<|reserved_special_token_78|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128087": {
+      "content": "<|reserved_special_token_79|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128088": {
+      "content": "<|reserved_special_token_80|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128089": {
+      "content": "<|reserved_special_token_81|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128090": {
+      "content": "<|reserved_special_token_82|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128091": {
+      "content": "<|reserved_special_token_83|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128092": {
+      "content": "<|reserved_special_token_84|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128093": {
+      "content": "<|reserved_special_token_85|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128094": {
+      "content": "<|reserved_special_token_86|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128095": {
+      "content": "<|reserved_special_token_87|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128096": {
+      "content": "<|reserved_special_token_88|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128097": {
+      "content": "<|reserved_special_token_89|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128098": {
+      "content": "<|reserved_special_token_90|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128099": {
+      "content": "<|reserved_special_token_91|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128100": {
+      "content": "<|reserved_special_token_92|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128101": {
+      "content": "<|reserved_special_token_93|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128102": {
+      "content": "<|reserved_special_token_94|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128103": {
+      "content": "<|reserved_special_token_95|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128104": {
+      "content": "<|reserved_special_token_96|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128105": {
+      "content": "<|reserved_special_token_97|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128106": {
+      "content": "<|reserved_special_token_98|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128107": {
+      "content": "<|reserved_special_token_99|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128108": {
+      "content": "<|reserved_special_token_100|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128109": {
+      "content": "<|reserved_special_token_101|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128110": {
+      "content": "<|reserved_special_token_102|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128111": {
+      "content": "<|reserved_special_token_103|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128112": {
+      "content": "<|reserved_special_token_104|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128113": {
+      "content": "<|reserved_special_token_105|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128114": {
+      "content": "<|reserved_special_token_106|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128115": {
+      "content": "<|reserved_special_token_107|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128116": {
+      "content": "<|reserved_special_token_108|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128117": {
+      "content": "<|reserved_special_token_109|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128118": {
+      "content": "<|reserved_special_token_110|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128119": {
+      "content": "<|reserved_special_token_111|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128120": {
+      "content": "<|reserved_special_token_112|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128121": {
+      "content": "<|reserved_special_token_113|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128122": {
+      "content": "<|reserved_special_token_114|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128123": {
+      "content": "<|reserved_special_token_115|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128124": {
+      "content": "<|reserved_special_token_116|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128125": {
+      "content": "<|reserved_special_token_117|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128126": {
+      "content": "<|reserved_special_token_118|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128127": {
+      "content": "<|reserved_special_token_119|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128128": {
+      "content": "<|reserved_special_token_120|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128129": {
+      "content": "<|reserved_special_token_121|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128130": {
+      "content": "<|reserved_special_token_122|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128131": {
+      "content": "<|reserved_special_token_123|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128132": {
+      "content": "<|reserved_special_token_124|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128133": {
+      "content": "<|reserved_special_token_125|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128134": {
+      "content": "<|reserved_special_token_126|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128135": {
+      "content": "<|reserved_special_token_127|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128136": {
+      "content": "<|reserved_special_token_128|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128137": {
+      "content": "<|reserved_special_token_129|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128138": {
+      "content": "<|reserved_special_token_130|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128139": {
+      "content": "<|reserved_special_token_131|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128140": {
+      "content": "<|reserved_special_token_132|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128141": {
+      "content": "<|reserved_special_token_133|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128142": {
+      "content": "<|reserved_special_token_134|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128143": {
+      "content": "<|reserved_special_token_135|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128144": {
+      "content": "<|reserved_special_token_136|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128145": {
+      "content": "<|reserved_special_token_137|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128146": {
+      "content": "<|reserved_special_token_138|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128147": {
+      "content": "<|reserved_special_token_139|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128148": {
+      "content": "<|reserved_special_token_140|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128149": {
+      "content": "<|reserved_special_token_141|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128150": {
+      "content": "<|reserved_special_token_142|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128151": {
+      "content": "<|reserved_special_token_143|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128152": {
+      "content": "<|reserved_special_token_144|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128153": {
+      "content": "<|reserved_special_token_145|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128154": {
+      "content": "<|reserved_special_token_146|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128155": {
+      "content": "<|reserved_special_token_147|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128156": {
+      "content": "<|reserved_special_token_148|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128157": {
+      "content": "<|reserved_special_token_149|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128158": {
+      "content": "<|reserved_special_token_150|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128159": {
+      "content": "<|reserved_special_token_151|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128160": {
+      "content": "<|reserved_special_token_152|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128161": {
+      "content": "<|reserved_special_token_153|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128162": {
+      "content": "<|reserved_special_token_154|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128163": {
+      "content": "<|reserved_special_token_155|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128164": {
+      "content": "<|reserved_special_token_156|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128165": {
+      "content": "<|reserved_special_token_157|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128166": {
+      "content": "<|reserved_special_token_158|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128167": {
+      "content": "<|reserved_special_token_159|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128168": {
+      "content": "<|reserved_special_token_160|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128169": {
+      "content": "<|reserved_special_token_161|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128170": {
+      "content": "<|reserved_special_token_162|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128171": {
+      "content": "<|reserved_special_token_163|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128172": {
+      "content": "<|reserved_special_token_164|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128173": {
+      "content": "<|reserved_special_token_165|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128174": {
+      "content": "<|reserved_special_token_166|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128175": {
+      "content": "<|reserved_special_token_167|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128176": {
+      "content": "<|reserved_special_token_168|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128177": {
+      "content": "<|reserved_special_token_169|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128178": {
+      "content": "<|reserved_special_token_170|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128179": {
+      "content": "<|reserved_special_token_171|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128180": {
+      "content": "<|reserved_special_token_172|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128181": {
+      "content": "<|reserved_special_token_173|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128182": {
+      "content": "<|reserved_special_token_174|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128183": {
+      "content": "<|reserved_special_token_175|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128184": {
+      "content": "<|reserved_special_token_176|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128185": {
+      "content": "<|reserved_special_token_177|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128186": {
+      "content": "<|reserved_special_token_178|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128187": {
+      "content": "<|reserved_special_token_179|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128188": {
+      "content": "<|reserved_special_token_180|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128189": {
+      "content": "<|reserved_special_token_181|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128190": {
+      "content": "<|reserved_special_token_182|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128191": {
+      "content": "<|reserved_special_token_183|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128192": {
+      "content": "<|reserved_special_token_184|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128193": {
+      "content": "<|reserved_special_token_185|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128194": {
+      "content": "<|reserved_special_token_186|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128195": {
+      "content": "<|reserved_special_token_187|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128196": {
+      "content": "<|reserved_special_token_188|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128197": {
+      "content": "<|reserved_special_token_189|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128198": {
+      "content": "<|reserved_special_token_190|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128199": {
+      "content": "<|reserved_special_token_191|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128200": {
+      "content": "<|reserved_special_token_192|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128201": {
+      "content": "<|reserved_special_token_193|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128202": {
+      "content": "<|reserved_special_token_194|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128203": {
+      "content": "<|reserved_special_token_195|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128204": {
+      "content": "<|reserved_special_token_196|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128205": {
+      "content": "<|reserved_special_token_197|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128206": {
+      "content": "<|reserved_special_token_198|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128207": {
+      "content": "<|reserved_special_token_199|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128208": {
+      "content": "<|reserved_special_token_200|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128209": {
+      "content": "<|reserved_special_token_201|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128210": {
+      "content": "<|reserved_special_token_202|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128211": {
+      "content": "<|reserved_special_token_203|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128212": {
+      "content": "<|reserved_special_token_204|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128213": {
+      "content": "<|reserved_special_token_205|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128214": {
+      "content": "<|reserved_special_token_206|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128215": {
+      "content": "<|reserved_special_token_207|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128216": {
+      "content": "<|reserved_special_token_208|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128217": {
+      "content": "<|reserved_special_token_209|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128218": {
+      "content": "<|reserved_special_token_210|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128219": {
+      "content": "<|reserved_special_token_211|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128220": {
+      "content": "<|reserved_special_token_212|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128221": {
+      "content": "<|reserved_special_token_213|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128222": {
+      "content": "<|reserved_special_token_214|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128223": {
+      "content": "<|reserved_special_token_215|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128224": {
+      "content": "<|reserved_special_token_216|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128225": {
+      "content": "<|reserved_special_token_217|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128226": {
+      "content": "<|reserved_special_token_218|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128227": {
+      "content": "<|reserved_special_token_219|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128228": {
+      "content": "<|reserved_special_token_220|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128229": {
+      "content": "<|reserved_special_token_221|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128230": {
+      "content": "<|reserved_special_token_222|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128231": {
+      "content": "<|reserved_special_token_223|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128232": {
+      "content": "<|reserved_special_token_224|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128233": {
+      "content": "<|reserved_special_token_225|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128234": {
+      "content": "<|reserved_special_token_226|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128235": {
+      "content": "<|reserved_special_token_227|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128236": {
+      "content": "<|reserved_special_token_228|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128237": {
+      "content": "<|reserved_special_token_229|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128238": {
+      "content": "<|reserved_special_token_230|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128239": {
+      "content": "<|reserved_special_token_231|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128240": {
+      "content": "<|reserved_special_token_232|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128241": {
+      "content": "<|reserved_special_token_233|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128242": {
+      "content": "<|reserved_special_token_234|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128243": {
+      "content": "<|reserved_special_token_235|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128244": {
+      "content": "<|reserved_special_token_236|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128245": {
+      "content": "<|reserved_special_token_237|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128246": {
+      "content": "<|reserved_special_token_238|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128247": {
+      "content": "<|reserved_special_token_239|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128248": {
+      "content": "<|reserved_special_token_240|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128249": {
+      "content": "<|reserved_special_token_241|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128250": {
+      "content": "<|reserved_special_token_242|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128251": {
+      "content": "<|reserved_special_token_243|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128252": {
+      "content": "<|reserved_special_token_244|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128253": {
+      "content": "<|reserved_special_token_245|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128254": {
+      "content": "<|reserved_special_token_246|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128255": {
+      "content": "<|reserved_special_token_247|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|begin_of_text|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|eot_id|>",
+  "extra_special_tokens": {},
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "pad_token": "<|reserved_special_token_247|>",
+  "tokenizer_class": "PreTrainedTokenizer"
+}

hotpotqa_skill_paraphrase_4_base_3epoch/64_e1_3.0e-05/train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 1.0,
+    "total_flos": 3.9356831750946816e+17,
+    "train_loss": 3.96218955618414,
+    "train_runtime": 277.7455,
+    "train_samples": 2802,
+    "train_samples_per_second": 10.088,
+    "train_steps_per_second": 0.634
+}

hotpotqa_skill_paraphrase_4_base_3epoch/64_e1_3.0e-05/trainer_state.json

@@ -0,0 +1,288 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 176,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.028530670470756064,
+      "grad_norm": 98.07823944091797,
+      "learning_rate": 6.666666666666667e-06,
+      "loss": 11.8227,
+      "step": 5
+    },
+    {
+      "epoch": 0.05706134094151213,
+      "grad_norm": 48.8634033203125,
+      "learning_rate": 1.5e-05,
+      "loss": 10.705,
+      "step": 10
+    },
+    {
+      "epoch": 0.08559201141226819,
+      "grad_norm": 22.390178680419922,
+      "learning_rate": 2.3333333333333336e-05,
+      "loss": 7.2769,
+      "step": 15
+    },
+    {
+      "epoch": 0.11412268188302425,
+      "grad_norm": 11.743159294128418,
+      "learning_rate": 2.9997034946550984e-05,
+      "loss": 5.1852,
+      "step": 20
+    },
+    {
+      "epoch": 0.14265335235378032,
+      "grad_norm": 11.406346321105957,
+      "learning_rate": 2.98933811055296e-05,
+      "loss": 4.8695,
+      "step": 25
+    },
+    {
+      "epoch": 0.17118402282453637,
+      "grad_norm": 9.969062805175781,
+      "learning_rate": 2.9642644716285765e-05,
+      "loss": 4.1906,
+      "step": 30
+    },
+    {
+      "epoch": 0.19971469329529243,
+      "grad_norm": 6.9679951667785645,
+      "learning_rate": 2.924730197591674e-05,
+      "loss": 3.4173,
+      "step": 35
+    },
+    {
+      "epoch": 0.2282453637660485,
+      "grad_norm": 13.462809562683105,
+      "learning_rate": 2.871125717031052e-05,
+      "loss": 4.063,
+      "step": 40
+    },
+    {
+      "epoch": 0.25677603423680456,
+      "grad_norm": 11.700884819030762,
+      "learning_rate": 2.8039804116593743e-05,
+      "loss": 3.9671,
+      "step": 45
+    },
+    {
+      "epoch": 0.28530670470756064,
+      "grad_norm": 9.339689254760742,
+      "learning_rate": 2.7239573882987418e-05,
+      "loss": 3.4171,
+      "step": 50
+    },
+    {
+      "epoch": 0.31383737517831667,
+      "grad_norm": 7.5099921226501465,
+      "learning_rate": 2.6318469302373453e-05,
+      "loss": 3.5813,
+      "step": 55
+    },
+    {
+      "epoch": 0.34236804564907275,
+      "grad_norm": 6.536746025085449,
+      "learning_rate": 2.52855869262962e-05,
+      "loss": 3.5958,
+      "step": 60
+    },
+    {
+      "epoch": 0.37089871611982883,
+      "grad_norm": 8.016010284423828,
+      "learning_rate": 2.4151127190157864e-05,
+      "loss": 3.6182,
+      "step": 65
+    },
+    {
+      "epoch": 0.39942938659058486,
+      "grad_norm": 6.370848178863525,
+      "learning_rate": 2.2926293676789295e-05,
+      "loss": 3.6103,
+      "step": 70
+    },
+    {
+      "epoch": 0.42796005706134094,
+      "grad_norm": 8.359729766845703,
+      "learning_rate": 2.162318247323868e-05,
+      "loss": 3.5373,
+      "step": 75
+    },
+    {
+      "epoch": 0.456490727532097,
+      "grad_norm": 6.346542835235596,
+      "learning_rate": 2.0254662713457366e-05,
+      "loss": 3.3341,
+      "step": 80
+    },
+    {
+      "epoch": 0.48502139800285304,
+      "grad_norm": 13.426467895507812,
+      "learning_rate": 1.883424948660712e-05,
+      "loss": 3.3211,
+      "step": 85
+    },
+    {
+      "epoch": 0.5135520684736091,
+      "grad_norm": 7.283843994140625,
+      "learning_rate": 1.7375970366108225e-05,
+      "loss": 3.2597,
+      "step": 90
+    },
+    {
+      "epoch": 0.5420827389443652,
+      "grad_norm": 9.1486234664917,
+      "learning_rate": 1.5894226877547298e-05,
+      "loss": 3.4157,
+      "step": 95
+    },
+    {
+      "epoch": 0.5706134094151213,
+      "grad_norm": 8.91500186920166,
+      "learning_rate": 1.4403652273546118e-05,
+      "loss": 3.1861,
+      "step": 100
+    },
+    {
+      "epoch": 0.5991440798858774,
+      "grad_norm": 8.01885986328125,
+      "learning_rate": 1.2918967020163978e-05,
+      "loss": 3.2207,
+      "step": 105
+    },
+    {
+      "epoch": 0.6276747503566333,
+      "grad_norm": 13.87535285949707,
+      "learning_rate": 1.1454833422006428e-05,
+      "loss": 3.2422,
+      "step": 110
+    },
+    {
+      "epoch": 0.6562054208273894,
+      "grad_norm": 6.999363422393799,
+      "learning_rate": 1.0025710821718983e-05,
+      "loss": 3.2623,
+      "step": 115
+    },
+    {
+      "epoch": 0.6847360912981455,
+      "grad_norm": 12.511735916137695,
+      "learning_rate": 8.645712803872084e-06,
+      "loss": 3.4606,
+      "step": 120
+    },
+    {
+      "epoch": 0.7132667617689016,
+      "grad_norm": 7.841530799865723,
+      "learning_rate": 7.3284678134486685e-06,
+      "loss": 2.9957,
+      "step": 125
+    },
+    {
+      "epoch": 0.7417974322396577,
+      "grad_norm": 7.247511863708496,
+      "learning_rate": 6.086984565424345e-06,
+      "loss": 2.9632,
+      "step": 130
+    },
+    {
+      "epoch": 0.7703281027104137,
+      "grad_norm": 6.871656894683838,
+      "learning_rate": 4.933523574614447e-06,
+      "loss": 3.1,
+      "step": 135
+    },
+    {
+      "epoch": 0.7988587731811697,
+      "grad_norm": 9.417755126953125,
+      "learning_rate": 3.879476074520731e-06,
+      "loss": 2.9654,
+      "step": 140
+    },
+    {
+      "epoch": 0.8273894436519258,
+      "grad_norm": 6.606371879577637,
+      "learning_rate": 2.935251520938528e-06,
+      "loss": 2.7702,
+      "step": 145
+    },
+    {
+      "epoch": 0.8559201141226819,
+      "grad_norm": 10.26504898071289,
+      "learning_rate": 2.1101747913050855e-06,
+      "loss": 2.9336,
+      "step": 150
+    },
+    {
+      "epoch": 0.884450784593438,
+      "grad_norm": 10.80896282196045,
+      "learning_rate": 1.412394095017151e-06,
+      "loss": 3.0025,
+      "step": 155
+    },
+    {
+      "epoch": 0.912981455064194,
+      "grad_norm": 8.550090789794922,
+      "learning_rate": 8.488005041679842e-07,
+      "loss": 2.9469,
+      "step": 160
+    },
+    {
+      "epoch": 0.9415121255349501,
+      "grad_norm": 7.228885173797607,
+      "learning_rate": 4.2495989939384916e-07,
+      "loss": 3.1122,
+      "step": 165
+    },
+    {
+      "epoch": 0.9700427960057061,
+      "grad_norm": 7.973071575164795,
+      "learning_rate": 1.4505800291247207e-07,
+      "loss": 2.9233,
+      "step": 170
+    },
+    {
+      "epoch": 0.9985734664764622,
+      "grad_norm": 7.4301252365112305,
+      "learning_rate": 1.1859041590472352e-08,
+      "loss": 3.0535,
+      "step": 175
+    },
+    {
+      "epoch": 1.0,
+      "step": 176,
+      "total_flos": 3.9356831750946816e+17,
+      "train_loss": 3.96218955618414,
+      "train_runtime": 277.7455,
+      "train_samples_per_second": 10.088,
+      "train_steps_per_second": 0.634
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 176,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 3.9356831750946816e+17,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}

hotpotqa_skill_paraphrase_4_base_3epoch/64_e1_3.0e-05/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6862db60ffd6c2d4a1dce55316821419a6e148ca0aecc445e6cdfd3caffc17fe
+size 8081

hotpotqa_skill_paraphrase_4_instruct_3epoch/64_e1_3.0e-05/adapter_config.json

@@ -0,0 +1,39 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "meta-llama/Llama-3.1-8B-Instruct",
+  "bias": "none",
+  "corda_config": null,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": false,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 128,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 64,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "up_proj",
+    "q_proj",
+    "k_proj",
+    "v_proj",
+    "o_proj",
+    "down_proj",
+    "gate_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_rslora": false
+}

hotpotqa_skill_paraphrase_4_instruct_3epoch/64_e1_3.0e-05/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d1bac12a9912ed22832f19e576e20188109ebeffec16a0490193d4d1d0fecb43
+size 335605144

hotpotqa_skill_paraphrase_4_instruct_3epoch/64_e1_3.0e-05/all_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 1.0,
+    "total_flos": 3.941173665328005e+17,
+    "train_loss": 1.885472126643766,
+    "train_runtime": 151.7112,
+    "train_samples": 2802,
+    "train_samples_per_second": 18.469,
+    "train_steps_per_second": 1.16
+}

hotpotqa_skill_paraphrase_4_instruct_3epoch/64_e1_3.0e-05/chat_template.jinja

@@ -0,0 +1,109 @@
+{{- bos_token }}
+{%- if custom_tools is defined %}
+    {%- set tools = custom_tools %}
+{%- endif %}
+{%- if not tools_in_user_message is defined %}
+    {%- set tools_in_user_message = true %}
+{%- endif %}
+{%- if not date_string is defined %}
+    {%- set date_string = "26 Jul 2024" %}
+{%- endif %}
+{%- if not tools is defined %}
+    {%- set tools = none %}
+{%- endif %}
+
+{#- This block extracts the system message, so we can slot it into the right place. #}
+{%- if messages[0]['role'] == 'system' %}
+    {%- set system_message = messages[0]['content']|trim %}
+    {%- set messages = messages[1:] %}
+{%- else %}
+    {%- set system_message = "" %}
+{%- endif %}
+
+{#- System message + builtin tools #}
+{{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
+{%- if builtin_tools is defined or tools is not none %}
+    {{- "Environment: ipython\n" }}
+{%- endif %}
+{%- if builtin_tools is defined %}
+    {{- "Tools: " + builtin_tools | reject('equalto', 'code_interpreter') | join(", ") + "\n\n"}}
+{%- endif %}
+{{- "Cutting Knowledge Date: December 2023\n" }}
+{{- "Today Date: " + date_string + "\n\n" }}
+{%- if tools is not none and not tools_in_user_message %}
+    {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }}
+    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+    {{- "Do not use variables.\n\n" }}
+    {%- for t in tools %}
+        {{- t | tojson(indent=4) }}
+        {{- "\n\n" }}
+    {%- endfor %}
+{%- endif %}
+{{- system_message }}
+{{- "<|eot_id|>" }}
+
+{#- Custom tools are passed in a user message with some extra guidance #}
+{%- if tools_in_user_message and not tools is none %}
+    {#- Extract the first user message so we can plug it in here #}
+    {%- if messages | length != 0 %}
+        {%- set first_user_message = messages[0]['content']|trim %}
+        {%- set messages = messages[1:] %}
+    {%- else %}
+        {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
+{%- endif %}
+    {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}}
+    {{- "Given the following functions, please respond with a JSON for a function call " }}
+    {{- "with its proper arguments that best answers the given prompt.\n\n" }}
+    {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }}
+    {{- "Do not use variables.\n\n" }}
+    {%- for t in tools %}
+        {{- t | tojson(indent=4) }}
+        {{- "\n\n" }}
+    {%- endfor %}
+    {{- first_user_message + "<|eot_id|>"}}
+{%- endif %}
+
+{%- for message in messages %}
+    {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
+        {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }}
+    {%- elif 'tool_calls' in message %}
+        {%- if not message.tool_calls|length == 1 %}
+            {{- raise_exception("This model only supports single tool-calls at once!") }}
+        {%- endif %}
+        {%- set tool_call = message.tool_calls[0].function %}
+        {%- if builtin_tools is defined and tool_call.name in builtin_tools %}
+            {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
+            {{- "<|python_tag|>" + tool_call.name + ".call(" }}
+            {%- for arg_name, arg_val in tool_call.arguments | items %}
+                {{- arg_name + '="' + arg_val + '"' }}
+                {%- if not loop.last %}
+                    {{- ", " }}
+                {%- endif %}
+                {%- endfor %}
+            {{- ")" }}
+        {%- else  %}
+            {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
+            {{- '{"name": "' + tool_call.name + '", ' }}
+            {{- '"parameters": ' }}
+            {{- tool_call.arguments | tojson }}
+            {{- "}" }}
+        {%- endif %}
+        {%- if builtin_tools is defined %}
+            {#- This means we're in ipython mode #}
+            {{- "<|eom_id|>" }}
+        {%- else %}
+            {{- "<|eot_id|>" }}
+        {%- endif %}
+    {%- elif message.role == "tool" or message.role == "ipython" %}
+        {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }}
+        {%- if message.content is mapping or message.content is iterable %}
+            {{- message.content | tojson }}
+        {%- else %}
+            {{- message.content }}
+        {%- endif %}
+        {{- "<|eot_id|>" }}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
+{%- endif %}

hotpotqa_skill_paraphrase_4_instruct_3epoch/64_e1_3.0e-05/config.json

@@ -0,0 +1,39 @@
+{
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "eos_token_id": [
+    128001,
+    128008,
+    128009
+  ],
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": {
+    "factor": 8.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_type": "llama3"
+  },
+  "rope_theta": 500000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.52.4",
+  "use_cache": true,
+  "vocab_size": 128256
+}

hotpotqa_skill_paraphrase_4_instruct_3epoch/64_e1_3.0e-05/generation_config.json

@@ -0,0 +1,12 @@
+{
+  "bos_token_id": 128000,
+  "do_sample": true,
+  "eos_token_id": [
+    128001,
+    128008,
+    128009
+  ],
+  "temperature": 0.6,
+  "top_p": 0.9,
+  "transformers_version": "4.52.4"
+}

hotpotqa_skill_paraphrase_4_instruct_3epoch/64_e1_3.0e-05/special_tokens_map.json

@@ -0,0 +1,17 @@
+{
+  "bos_token": {
+    "content": "<|begin_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|eot_id|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<|reserved_special_token_247|>"
+}

hotpotqa_skill_paraphrase_4_instruct_3epoch/64_e1_3.0e-05/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b9e4e7fb171f92fd137b777cc2714bf87d11576700a1dcd7a399e7bbe39537b
+size 17209920

hotpotqa_skill_paraphrase_4_instruct_3epoch/64_e1_3.0e-05/tokenizer_config.json

@@ -0,0 +1,2063 @@
+{
+  "added_tokens_decoder": {
+    "128000": {
+      "content": "<|begin_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128001": {
+      "content": "<|end_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128002": {
+      "content": "<|reserved_special_token_0|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128003": {
+      "content": "<|reserved_special_token_1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128004": {
+      "content": "<|finetune_right_pad_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128005": {
+      "content": "<|reserved_special_token_2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128006": {
+      "content": "<|start_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128007": {
+      "content": "<|end_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128008": {
+      "content": "<|eom_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128009": {
+      "content": "<|eot_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128010": {
+      "content": "<|python_tag|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128011": {
+      "content": "<|reserved_special_token_3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128012": {
+      "content": "<|reserved_special_token_4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128013": {
+      "content": "<|reserved_special_token_5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128014": {
+      "content": "<|reserved_special_token_6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128015": {
+      "content": "<|reserved_special_token_7|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128016": {
+      "content": "<|reserved_special_token_8|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128017": {
+      "content": "<|reserved_special_token_9|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128018": {
+      "content": "<|reserved_special_token_10|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128019": {
+      "content": "<|reserved_special_token_11|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128020": {
+      "content": "<|reserved_special_token_12|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128021": {
+      "content": "<|reserved_special_token_13|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128022": {
+      "content": "<|reserved_special_token_14|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128023": {
+      "content": "<|reserved_special_token_15|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128024": {
+      "content": "<|reserved_special_token_16|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128025": {
+      "content": "<|reserved_special_token_17|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128026": {
+      "content": "<|reserved_special_token_18|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128027": {
+      "content": "<|reserved_special_token_19|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128028": {
+      "content": "<|reserved_special_token_20|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128029": {
+      "content": "<|reserved_special_token_21|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128030": {
+      "content": "<|reserved_special_token_22|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128031": {
+      "content": "<|reserved_special_token_23|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128032": {
+      "content": "<|reserved_special_token_24|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128033": {
+      "content": "<|reserved_special_token_25|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128034": {
+      "content": "<|reserved_special_token_26|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128035": {
+      "content": "<|reserved_special_token_27|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128036": {
+      "content": "<|reserved_special_token_28|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128037": {
+      "content": "<|reserved_special_token_29|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128038": {
+      "content": "<|reserved_special_token_30|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128039": {
+      "content": "<|reserved_special_token_31|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128040": {
+      "content": "<|reserved_special_token_32|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128041": {
+      "content": "<|reserved_special_token_33|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128042": {
+      "content": "<|reserved_special_token_34|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128043": {
+      "content": "<|reserved_special_token_35|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128044": {
+      "content": "<|reserved_special_token_36|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128045": {
+      "content": "<|reserved_special_token_37|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128046": {
+      "content": "<|reserved_special_token_38|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128047": {
+      "content": "<|reserved_special_token_39|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128048": {
+      "content": "<|reserved_special_token_40|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128049": {
+      "content": "<|reserved_special_token_41|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128050": {
+      "content": "<|reserved_special_token_42|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128051": {
+      "content": "<|reserved_special_token_43|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128052": {
+      "content": "<|reserved_special_token_44|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128053": {
+      "content": "<|reserved_special_token_45|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128054": {
+      "content": "<|reserved_special_token_46|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128055": {
+      "content": "<|reserved_special_token_47|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128056": {
+      "content": "<|reserved_special_token_48|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128057": {
+      "content": "<|reserved_special_token_49|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128058": {
+      "content": "<|reserved_special_token_50|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128059": {
+      "content": "<|reserved_special_token_51|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128060": {
+      "content": "<|reserved_special_token_52|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128061": {
+      "content": "<|reserved_special_token_53|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128062": {
+      "content": "<|reserved_special_token_54|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128063": {
+      "content": "<|reserved_special_token_55|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128064": {
+      "content": "<|reserved_special_token_56|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128065": {
+      "content": "<|reserved_special_token_57|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128066": {
+      "content": "<|reserved_special_token_58|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128067": {
+      "content": "<|reserved_special_token_59|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128068": {
+      "content": "<|reserved_special_token_60|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128069": {
+      "content": "<|reserved_special_token_61|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128070": {
+      "content": "<|reserved_special_token_62|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128071": {
+      "content": "<|reserved_special_token_63|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128072": {
+      "content": "<|reserved_special_token_64|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128073": {
+      "content": "<|reserved_special_token_65|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128074": {
+      "content": "<|reserved_special_token_66|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128075": {
+      "content": "<|reserved_special_token_67|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128076": {
+      "content": "<|reserved_special_token_68|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128077": {
+      "content": "<|reserved_special_token_69|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128078": {
+      "content": "<|reserved_special_token_70|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128079": {
+      "content": "<|reserved_special_token_71|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128080": {
+      "content": "<|reserved_special_token_72|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128081": {
+      "content": "<|reserved_special_token_73|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128082": {
+      "content": "<|reserved_special_token_74|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128083": {
+      "content": "<|reserved_special_token_75|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128084": {
+      "content": "<|reserved_special_token_76|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128085": {
+      "content": "<|reserved_special_token_77|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128086": {
+      "content": "<|reserved_special_token_78|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128087": {
+      "content": "<|reserved_special_token_79|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128088": {
+      "content": "<|reserved_special_token_80|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128089": {
+      "content": "<|reserved_special_token_81|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128090": {
+      "content": "<|reserved_special_token_82|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128091": {
+      "content": "<|reserved_special_token_83|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128092": {
+      "content": "<|reserved_special_token_84|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128093": {
+      "content": "<|reserved_special_token_85|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128094": {
+      "content": "<|reserved_special_token_86|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128095": {
+      "content": "<|reserved_special_token_87|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128096": {
+      "content": "<|reserved_special_token_88|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128097": {
+      "content": "<|reserved_special_token_89|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128098": {
+      "content": "<|reserved_special_token_90|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128099": {
+      "content": "<|reserved_special_token_91|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128100": {
+      "content": "<|reserved_special_token_92|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128101": {
+      "content": "<|reserved_special_token_93|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128102": {
+      "content": "<|reserved_special_token_94|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128103": {
+      "content": "<|reserved_special_token_95|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128104": {
+      "content": "<|reserved_special_token_96|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128105": {
+      "content": "<|reserved_special_token_97|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128106": {
+      "content": "<|reserved_special_token_98|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128107": {
+      "content": "<|reserved_special_token_99|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128108": {
+      "content": "<|reserved_special_token_100|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128109": {
+      "content": "<|reserved_special_token_101|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128110": {
+      "content": "<|reserved_special_token_102|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128111": {
+      "content": "<|reserved_special_token_103|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128112": {
+      "content": "<|reserved_special_token_104|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128113": {
+      "content": "<|reserved_special_token_105|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128114": {
+      "content": "<|reserved_special_token_106|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128115": {
+      "content": "<|reserved_special_token_107|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128116": {
+      "content": "<|reserved_special_token_108|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128117": {
+      "content": "<|reserved_special_token_109|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128118": {
+      "content": "<|reserved_special_token_110|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128119": {
+      "content": "<|reserved_special_token_111|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128120": {
+      "content": "<|reserved_special_token_112|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128121": {
+      "content": "<|reserved_special_token_113|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128122": {
+      "content": "<|reserved_special_token_114|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128123": {
+      "content": "<|reserved_special_token_115|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128124": {
+      "content": "<|reserved_special_token_116|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128125": {
+      "content": "<|reserved_special_token_117|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128126": {
+      "content": "<|reserved_special_token_118|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128127": {
+      "content": "<|reserved_special_token_119|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128128": {
+      "content": "<|reserved_special_token_120|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128129": {
+      "content": "<|reserved_special_token_121|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128130": {
+      "content": "<|reserved_special_token_122|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128131": {
+      "content": "<|reserved_special_token_123|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128132": {
+      "content": "<|reserved_special_token_124|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128133": {
+      "content": "<|reserved_special_token_125|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128134": {
+      "content": "<|reserved_special_token_126|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128135": {
+      "content": "<|reserved_special_token_127|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128136": {
+      "content": "<|reserved_special_token_128|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128137": {
+      "content": "<|reserved_special_token_129|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128138": {
+      "content": "<|reserved_special_token_130|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128139": {
+      "content": "<|reserved_special_token_131|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128140": {
+      "content": "<|reserved_special_token_132|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128141": {
+      "content": "<|reserved_special_token_133|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128142": {
+      "content": "<|reserved_special_token_134|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128143": {
+      "content": "<|reserved_special_token_135|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128144": {
+      "content": "<|reserved_special_token_136|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128145": {
+      "content": "<|reserved_special_token_137|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128146": {
+      "content": "<|reserved_special_token_138|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128147": {
+      "content": "<|reserved_special_token_139|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128148": {
+      "content": "<|reserved_special_token_140|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128149": {
+      "content": "<|reserved_special_token_141|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128150": {
+      "content": "<|reserved_special_token_142|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128151": {
+      "content": "<|reserved_special_token_143|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128152": {
+      "content": "<|reserved_special_token_144|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128153": {
+      "content": "<|reserved_special_token_145|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128154": {
+      "content": "<|reserved_special_token_146|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128155": {
+      "content": "<|reserved_special_token_147|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128156": {
+      "content": "<|reserved_special_token_148|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128157": {
+      "content": "<|reserved_special_token_149|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128158": {
+      "content": "<|reserved_special_token_150|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128159": {
+      "content": "<|reserved_special_token_151|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128160": {
+      "content": "<|reserved_special_token_152|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128161": {
+      "content": "<|reserved_special_token_153|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128162": {
+      "content": "<|reserved_special_token_154|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128163": {
+      "content": "<|reserved_special_token_155|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128164": {
+      "content": "<|reserved_special_token_156|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128165": {
+      "content": "<|reserved_special_token_157|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128166": {
+      "content": "<|reserved_special_token_158|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128167": {
+      "content": "<|reserved_special_token_159|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128168": {
+      "content": "<|reserved_special_token_160|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128169": {
+      "content": "<|reserved_special_token_161|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128170": {
+      "content": "<|reserved_special_token_162|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128171": {
+      "content": "<|reserved_special_token_163|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128172": {
+      "content": "<|reserved_special_token_164|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128173": {
+      "content": "<|reserved_special_token_165|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128174": {
+      "content": "<|reserved_special_token_166|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128175": {
+      "content": "<|reserved_special_token_167|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128176": {
+      "content": "<|reserved_special_token_168|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128177": {
+      "content": "<|reserved_special_token_169|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128178": {
+      "content": "<|reserved_special_token_170|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128179": {
+      "content": "<|reserved_special_token_171|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128180": {
+      "content": "<|reserved_special_token_172|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128181": {
+      "content": "<|reserved_special_token_173|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128182": {
+      "content": "<|reserved_special_token_174|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128183": {
+      "content": "<|reserved_special_token_175|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128184": {
+      "content": "<|reserved_special_token_176|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128185": {
+      "content": "<|reserved_special_token_177|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128186": {
+      "content": "<|reserved_special_token_178|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128187": {
+      "content": "<|reserved_special_token_179|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128188": {
+      "content": "<|reserved_special_token_180|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128189": {
+      "content": "<|reserved_special_token_181|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128190": {
+      "content": "<|reserved_special_token_182|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128191": {
+      "content": "<|reserved_special_token_183|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128192": {
+      "content": "<|reserved_special_token_184|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128193": {
+      "content": "<|reserved_special_token_185|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128194": {
+      "content": "<|reserved_special_token_186|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128195": {
+      "content": "<|reserved_special_token_187|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128196": {
+      "content": "<|reserved_special_token_188|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128197": {
+      "content": "<|reserved_special_token_189|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128198": {
+      "content": "<|reserved_special_token_190|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128199": {
+      "content": "<|reserved_special_token_191|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128200": {
+      "content": "<|reserved_special_token_192|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128201": {
+      "content": "<|reserved_special_token_193|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128202": {
+      "content": "<|reserved_special_token_194|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128203": {
+      "content": "<|reserved_special_token_195|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128204": {
+      "content": "<|reserved_special_token_196|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128205": {
+      "content": "<|reserved_special_token_197|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128206": {
+      "content": "<|reserved_special_token_198|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128207": {
+      "content": "<|reserved_special_token_199|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128208": {
+      "content": "<|reserved_special_token_200|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128209": {
+      "content": "<|reserved_special_token_201|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128210": {
+      "content": "<|reserved_special_token_202|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128211": {
+      "content": "<|reserved_special_token_203|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128212": {
+      "content": "<|reserved_special_token_204|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128213": {
+      "content": "<|reserved_special_token_205|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128214": {
+      "content": "<|reserved_special_token_206|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128215": {
+      "content": "<|reserved_special_token_207|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128216": {
+      "content": "<|reserved_special_token_208|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128217": {
+      "content": "<|reserved_special_token_209|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128218": {
+      "content": "<|reserved_special_token_210|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128219": {
+      "content": "<|reserved_special_token_211|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128220": {
+      "content": "<|reserved_special_token_212|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128221": {
+      "content": "<|reserved_special_token_213|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128222": {
+      "content": "<|reserved_special_token_214|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128223": {
+      "content": "<|reserved_special_token_215|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128224": {
+      "content": "<|reserved_special_token_216|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128225": {
+      "content": "<|reserved_special_token_217|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128226": {
+      "content": "<|reserved_special_token_218|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128227": {
+      "content": "<|reserved_special_token_219|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128228": {
+      "content": "<|reserved_special_token_220|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128229": {
+      "content": "<|reserved_special_token_221|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128230": {
+      "content": "<|reserved_special_token_222|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128231": {
+      "content": "<|reserved_special_token_223|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128232": {
+      "content": "<|reserved_special_token_224|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128233": {
+      "content": "<|reserved_special_token_225|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128234": {
+      "content": "<|reserved_special_token_226|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128235": {
+      "content": "<|reserved_special_token_227|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128236": {
+      "content": "<|reserved_special_token_228|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128237": {
+      "content": "<|reserved_special_token_229|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128238": {
+      "content": "<|reserved_special_token_230|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128239": {
+      "content": "<|reserved_special_token_231|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128240": {
+      "content": "<|reserved_special_token_232|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128241": {
+      "content": "<|reserved_special_token_233|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128242": {
+      "content": "<|reserved_special_token_234|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128243": {
+      "content": "<|reserved_special_token_235|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128244": {
+      "content": "<|reserved_special_token_236|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128245": {
+      "content": "<|reserved_special_token_237|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128246": {
+      "content": "<|reserved_special_token_238|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128247": {
+      "content": "<|reserved_special_token_239|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128248": {
+      "content": "<|reserved_special_token_240|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128249": {
+      "content": "<|reserved_special_token_241|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128250": {
+      "content": "<|reserved_special_token_242|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128251": {
+      "content": "<|reserved_special_token_243|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128252": {
+      "content": "<|reserved_special_token_244|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128253": {
+      "content": "<|reserved_special_token_245|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128254": {
+      "content": "<|reserved_special_token_246|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128255": {
+      "content": "<|reserved_special_token_247|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|begin_of_text|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|eot_id|>",
+  "extra_special_tokens": {},
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "pad_token": "<|reserved_special_token_247|>",
+  "tokenizer_class": "PreTrainedTokenizer"
+}

hotpotqa_skill_paraphrase_4_instruct_3epoch/64_e1_3.0e-05/train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 1.0,
+    "total_flos": 3.941173665328005e+17,
+    "train_loss": 1.885472126643766,
+    "train_runtime": 151.7112,
+    "train_samples": 2802,
+    "train_samples_per_second": 18.469,
+    "train_steps_per_second": 1.16
+}

hotpotqa_skill_paraphrase_4_instruct_3epoch/64_e1_3.0e-05/trainer_state.json

@@ -0,0 +1,288 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 176,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.02849002849002849,
+      "grad_norm": 41.54158401489258,
+      "learning_rate": 6.666666666666667e-06,
+      "loss": 4.6986,
+      "step": 5
+    },
+    {
+      "epoch": 0.05698005698005698,
+      "grad_norm": 22.764928817749023,
+      "learning_rate": 1.5e-05,
+      "loss": 4.3109,
+      "step": 10
+    },
+    {
+      "epoch": 0.08547008547008547,
+      "grad_norm": 10.573023796081543,
+      "learning_rate": 2.3333333333333336e-05,
+      "loss": 3.3624,
+      "step": 15
+    },
+    {
+      "epoch": 0.11396011396011396,
+      "grad_norm": 6.423072338104248,
+      "learning_rate": 2.9997034946550984e-05,
+      "loss": 2.5337,
+      "step": 20
+    },
+    {
+      "epoch": 0.14245014245014245,
+      "grad_norm": 6.361787796020508,
+      "learning_rate": 2.98933811055296e-05,
+      "loss": 2.4237,
+      "step": 25
+    },
+    {
+      "epoch": 0.17094017094017094,
+      "grad_norm": 4.749721527099609,
+      "learning_rate": 2.9642644716285765e-05,
+      "loss": 2.0534,
+      "step": 30
+    },
+    {
+      "epoch": 0.19943019943019943,
+      "grad_norm": 3.451364517211914,
+      "learning_rate": 2.924730197591674e-05,
+      "loss": 1.7088,
+      "step": 35
+    },
+    {
+      "epoch": 0.22792022792022792,
+      "grad_norm": 8.959653854370117,
+      "learning_rate": 2.871125717031052e-05,
+      "loss": 2.0079,
+      "step": 40
+    },
+    {
+      "epoch": 0.2564102564102564,
+      "grad_norm": 7.419877052307129,
+      "learning_rate": 2.8039804116593743e-05,
+      "loss": 1.9297,
+      "step": 45
+    },
+    {
+      "epoch": 0.2849002849002849,
+      "grad_norm": 4.485551357269287,
+      "learning_rate": 2.7239573882987418e-05,
+      "loss": 1.7006,
+      "step": 50
+    },
+    {
+      "epoch": 0.31339031339031337,
+      "grad_norm": 4.072010517120361,
+      "learning_rate": 2.6318469302373453e-05,
+      "loss": 1.7362,
+      "step": 55
+    },
+    {
+      "epoch": 0.3418803418803419,
+      "grad_norm": 3.5165116786956787,
+      "learning_rate": 2.52855869262962e-05,
+      "loss": 1.791,
+      "step": 60
+    },
+    {
+      "epoch": 0.37037037037037035,
+      "grad_norm": 3.970125198364258,
+      "learning_rate": 2.4151127190157864e-05,
+      "loss": 1.8049,
+      "step": 65
+    },
+    {
+      "epoch": 0.39886039886039887,
+      "grad_norm": 3.4378178119659424,
+      "learning_rate": 2.2926293676789295e-05,
+      "loss": 1.7725,
+      "step": 70
+    },
+    {
+      "epoch": 0.42735042735042733,
+      "grad_norm": 4.609078407287598,
+      "learning_rate": 2.162318247323868e-05,
+      "loss": 1.7165,
+      "step": 75
+    },
+    {
+      "epoch": 0.45584045584045585,
+      "grad_norm": 3.251359224319458,
+      "learning_rate": 2.0254662713457366e-05,
+      "loss": 1.6442,
+      "step": 80
+    },
+    {
+      "epoch": 0.4843304843304843,
+      "grad_norm": 5.737546443939209,
+      "learning_rate": 1.883424948660712e-05,
+      "loss": 1.6232,
+      "step": 85
+    },
+    {
+      "epoch": 0.5128205128205128,
+      "grad_norm": 4.057260036468506,
+      "learning_rate": 1.7375970366108225e-05,
+      "loss": 1.5908,
+      "step": 90
+    },
+    {
+      "epoch": 0.5413105413105413,
+      "grad_norm": 4.2831315994262695,
+      "learning_rate": 1.5894226877547298e-05,
+      "loss": 1.7128,
+      "step": 95
+    },
+    {
+      "epoch": 0.5698005698005698,
+      "grad_norm": 3.8855299949645996,
+      "learning_rate": 1.4403652273546118e-05,
+      "loss": 1.5593,
+      "step": 100
+    },
+    {
+      "epoch": 0.5982905982905983,
+      "grad_norm": 3.7846293449401855,
+      "learning_rate": 1.2918967020163978e-05,
+      "loss": 1.5821,
+      "step": 105
+    },
+    {
+      "epoch": 0.6267806267806267,
+      "grad_norm": 6.312170505523682,
+      "learning_rate": 1.1454833422006428e-05,
+      "loss": 1.5836,
+      "step": 110
+    },
+    {
+      "epoch": 0.6552706552706553,
+      "grad_norm": 3.63012433052063,
+      "learning_rate": 1.0025710821718983e-05,
+      "loss": 1.584,
+      "step": 115
+    },
+    {
+      "epoch": 0.6837606837606838,
+      "grad_norm": 5.612448692321777,
+      "learning_rate": 8.645712803872084e-06,
+      "loss": 1.7054,
+      "step": 120
+    },
+    {
+      "epoch": 0.7122507122507122,
+      "grad_norm": 3.9694271087646484,
+      "learning_rate": 7.3284678134486685e-06,
+      "loss": 1.4795,
+      "step": 125
+    },
+    {
+      "epoch": 0.7407407407407407,
+      "grad_norm": 3.8035147190093994,
+      "learning_rate": 6.086984565424345e-06,
+      "loss": 1.4751,
+      "step": 130
+    },
+    {
+      "epoch": 0.7692307692307693,
+      "grad_norm": 3.422909736633301,
+      "learning_rate": 4.933523574614447e-06,
+      "loss": 1.5283,
+      "step": 135
+    },
+    {
+      "epoch": 0.7977207977207977,
+      "grad_norm": 4.461482524871826,
+      "learning_rate": 3.879476074520731e-06,
+      "loss": 1.4482,
+      "step": 140
+    },
+    {
+      "epoch": 0.8262108262108262,
+      "grad_norm": 3.229851722717285,
+      "learning_rate": 2.935251520938528e-06,
+      "loss": 1.3772,
+      "step": 145
+    },
+    {
+      "epoch": 0.8547008547008547,
+      "grad_norm": 4.533471584320068,
+      "learning_rate": 2.1101747913050855e-06,
+      "loss": 1.4373,
+      "step": 150
+    },
+    {
+      "epoch": 0.8831908831908832,
+      "grad_norm": 4.994824409484863,
+      "learning_rate": 1.412394095017151e-06,
+      "loss": 1.4558,
+      "step": 155
+    },
+    {
+      "epoch": 0.9116809116809117,
+      "grad_norm": 3.992816925048828,
+      "learning_rate": 8.488005041679842e-07,
+      "loss": 1.4463,
+      "step": 160
+    },
+    {
+      "epoch": 0.9401709401709402,
+      "grad_norm": 3.7581281661987305,
+      "learning_rate": 4.2495989939384916e-07,
+      "loss": 1.5126,
+      "step": 165
+    },
+    {
+      "epoch": 0.9686609686609686,
+      "grad_norm": 3.792056083679199,
+      "learning_rate": 1.4505800291247207e-07,
+      "loss": 1.4521,
+      "step": 170
+    },
+    {
+      "epoch": 0.9971509971509972,
+      "grad_norm": 4.3386077880859375,
+      "learning_rate": 1.1859041590472352e-08,
+      "loss": 1.4799,
+      "step": 175
+    },
+    {
+      "epoch": 1.0,
+      "step": 176,
+      "total_flos": 3.941173665328005e+17,
+      "train_loss": 1.885472126643766,
+      "train_runtime": 151.7112,
+      "train_samples_per_second": 18.469,
+      "train_steps_per_second": 1.16
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 176,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 3.941173665328005e+17,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}

hotpotqa_skill_paraphrase_4_instruct_3epoch/64_e1_3.0e-05/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:82e6575c7eaad6eeba26b1b2ae8bdc165ab078951c886c5486c4becfad525d8e
+size 8081

hotpotqa_test_knowledge_50_base/0_128_e3_3e-5/README.md

@@ -0,0 +1,63 @@
+---
+library_name: peft
+license: llama3.1
+base_model: meta-llama/Llama-3.1-8B
+tags:
+- alignment-handbook
+- generated_from_trainer
+datasets:
+- data/hotpotqa/test/processed/knowledge_50
+model-index:
+- name: 0_128_e3_3e-5
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# 0_128_e3_3e-5
+
+This model is a fine-tuned version of [meta-llama/Llama-3.1-8B](https://huggingface.co/meta-llama/Llama-3.1-8B) on the data/hotpotqa/test/processed/knowledge_50 dataset.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 3e-05
+- train_batch_size: 2
+- eval_batch_size: 1
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 4
+- gradient_accumulation_steps: 4
+- total_train_batch_size: 32
+- total_eval_batch_size: 4
+- optimizer: Use adamw_torch with betas=(0.9,0.95) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.05
+- num_epochs: 3.0
+
+### Training results
+
+
+
+### Framework versions
+
+- PEFT 0.15.2
+- Transformers 4.52.4
+- Pytorch 2.7.0+cu126
+- Datasets 3.6.0
+- Tokenizers 0.21.2

hotpotqa_test_knowledge_50_base/0_128_e3_3e-5/adapter_config.json

@@ -0,0 +1,39 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "meta-llama/Llama-3.1-8B",
+  "bias": "none",
+  "corda_config": null,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 256,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 128,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "q_proj",
+    "gate_proj",
+    "v_proj",
+    "o_proj",
+    "down_proj",
+    "up_proj",
+    "k_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_rslora": false
+}

hotpotqa_test_knowledge_50_base/0_128_e3_3e-5/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e0fa3bcf3a2a32b41ca909dabbb5384252c18ec8b72131d784cda5fceaf33253
+size 671150064

hotpotqa_test_knowledge_50_base/0_128_e3_3e-5/all_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "total_flos": 1.8970279091854377e+18,
+    "train_loss": 0.46635930327050884,
+    "train_runtime": 1484.7838,
+    "train_samples": 14795,
+    "train_samples_per_second": 29.893,
+    "train_steps_per_second": 0.935
+}

hotpotqa_test_knowledge_50_base/0_128_e3_3e-5/config.json

@@ -0,0 +1,35 @@
+{
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "eos_token_id": 128001,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": {
+    "factor": 8.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_type": "llama3"
+  },
+  "rope_theta": 500000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.52.4",
+  "use_cache": true,
+  "vocab_size": 128256
+}

hotpotqa_test_knowledge_50_base/0_128_e3_3e-5/special_tokens_map.json

@@ -0,0 +1,17 @@
+{
+  "bos_token": {
+    "content": "<|begin_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|end_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<|reserved_special_token_247|>"
+}

hotpotqa_test_knowledge_50_base/0_128_e3_3e-5/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:384a7e7c676f7be2e5d2e8449c508be9b00e5b18c5b3c39ebc626e96b3f4b988
+size 17210019

hotpotqa_test_knowledge_50_base/0_128_e3_3e-5/tokenizer_config.json

@@ -0,0 +1,2063 @@
+{
+  "added_tokens_decoder": {
+    "128000": {
+      "content": "<|begin_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128001": {
+      "content": "<|end_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128002": {
+      "content": "<|reserved_special_token_0|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128003": {
+      "content": "<|reserved_special_token_1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128004": {
+      "content": "<|finetune_right_pad_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128005": {
+      "content": "<|reserved_special_token_2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128006": {
+      "content": "<|start_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128007": {
+      "content": "<|end_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128008": {
+      "content": "<|eom_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128009": {
+      "content": "<|eot_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128010": {
+      "content": "<|python_tag|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128011": {
+      "content": "<|reserved_special_token_3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128012": {
+      "content": "<|reserved_special_token_4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128013": {
+      "content": "<|reserved_special_token_5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128014": {
+      "content": "<|reserved_special_token_6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128015": {
+      "content": "<|reserved_special_token_7|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128016": {
+      "content": "<|reserved_special_token_8|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128017": {
+      "content": "<|reserved_special_token_9|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128018": {
+      "content": "<|reserved_special_token_10|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128019": {
+      "content": "<|reserved_special_token_11|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128020": {
+      "content": "<|reserved_special_token_12|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128021": {
+      "content": "<|reserved_special_token_13|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128022": {
+      "content": "<|reserved_special_token_14|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128023": {
+      "content": "<|reserved_special_token_15|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128024": {
+      "content": "<|reserved_special_token_16|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128025": {
+      "content": "<|reserved_special_token_17|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128026": {
+      "content": "<|reserved_special_token_18|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128027": {
+      "content": "<|reserved_special_token_19|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128028": {
+      "content": "<|reserved_special_token_20|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128029": {
+      "content": "<|reserved_special_token_21|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128030": {
+      "content": "<|reserved_special_token_22|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128031": {
+      "content": "<|reserved_special_token_23|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128032": {
+      "content": "<|reserved_special_token_24|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128033": {
+      "content": "<|reserved_special_token_25|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128034": {
+      "content": "<|reserved_special_token_26|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128035": {
+      "content": "<|reserved_special_token_27|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128036": {
+      "content": "<|reserved_special_token_28|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128037": {
+      "content": "<|reserved_special_token_29|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128038": {
+      "content": "<|reserved_special_token_30|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128039": {
+      "content": "<|reserved_special_token_31|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128040": {
+      "content": "<|reserved_special_token_32|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128041": {
+      "content": "<|reserved_special_token_33|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128042": {
+      "content": "<|reserved_special_token_34|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128043": {
+      "content": "<|reserved_special_token_35|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128044": {
+      "content": "<|reserved_special_token_36|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128045": {
+      "content": "<|reserved_special_token_37|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128046": {
+      "content": "<|reserved_special_token_38|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128047": {
+      "content": "<|reserved_special_token_39|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128048": {
+      "content": "<|reserved_special_token_40|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128049": {
+      "content": "<|reserved_special_token_41|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128050": {
+      "content": "<|reserved_special_token_42|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128051": {
+      "content": "<|reserved_special_token_43|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128052": {
+      "content": "<|reserved_special_token_44|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128053": {
+      "content": "<|reserved_special_token_45|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128054": {
+      "content": "<|reserved_special_token_46|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128055": {
+      "content": "<|reserved_special_token_47|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128056": {
+      "content": "<|reserved_special_token_48|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128057": {
+      "content": "<|reserved_special_token_49|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128058": {
+      "content": "<|reserved_special_token_50|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128059": {
+      "content": "<|reserved_special_token_51|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128060": {
+      "content": "<|reserved_special_token_52|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128061": {
+      "content": "<|reserved_special_token_53|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128062": {
+      "content": "<|reserved_special_token_54|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128063": {
+      "content": "<|reserved_special_token_55|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128064": {
+      "content": "<|reserved_special_token_56|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128065": {
+      "content": "<|reserved_special_token_57|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128066": {
+      "content": "<|reserved_special_token_58|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128067": {
+      "content": "<|reserved_special_token_59|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128068": {
+      "content": "<|reserved_special_token_60|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128069": {
+      "content": "<|reserved_special_token_61|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128070": {
+      "content": "<|reserved_special_token_62|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128071": {
+      "content": "<|reserved_special_token_63|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128072": {
+      "content": "<|reserved_special_token_64|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128073": {
+      "content": "<|reserved_special_token_65|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128074": {
+      "content": "<|reserved_special_token_66|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128075": {
+      "content": "<|reserved_special_token_67|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128076": {
+      "content": "<|reserved_special_token_68|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128077": {
+      "content": "<|reserved_special_token_69|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128078": {
+      "content": "<|reserved_special_token_70|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128079": {
+      "content": "<|reserved_special_token_71|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128080": {
+      "content": "<|reserved_special_token_72|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128081": {
+      "content": "<|reserved_special_token_73|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128082": {
+      "content": "<|reserved_special_token_74|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128083": {
+      "content": "<|reserved_special_token_75|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128084": {
+      "content": "<|reserved_special_token_76|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128085": {
+      "content": "<|reserved_special_token_77|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128086": {
+      "content": "<|reserved_special_token_78|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128087": {
+      "content": "<|reserved_special_token_79|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128088": {
+      "content": "<|reserved_special_token_80|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128089": {
+      "content": "<|reserved_special_token_81|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128090": {
+      "content": "<|reserved_special_token_82|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128091": {
+      "content": "<|reserved_special_token_83|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128092": {
+      "content": "<|reserved_special_token_84|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128093": {
+      "content": "<|reserved_special_token_85|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128094": {
+      "content": "<|reserved_special_token_86|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128095": {
+      "content": "<|reserved_special_token_87|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128096": {
+      "content": "<|reserved_special_token_88|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128097": {
+      "content": "<|reserved_special_token_89|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128098": {
+      "content": "<|reserved_special_token_90|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128099": {
+      "content": "<|reserved_special_token_91|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128100": {
+      "content": "<|reserved_special_token_92|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128101": {
+      "content": "<|reserved_special_token_93|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128102": {
+      "content": "<|reserved_special_token_94|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128103": {
+      "content": "<|reserved_special_token_95|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128104": {
+      "content": "<|reserved_special_token_96|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128105": {
+      "content": "<|reserved_special_token_97|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128106": {
+      "content": "<|reserved_special_token_98|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128107": {
+      "content": "<|reserved_special_token_99|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128108": {
+      "content": "<|reserved_special_token_100|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128109": {
+      "content": "<|reserved_special_token_101|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128110": {
+      "content": "<|reserved_special_token_102|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128111": {
+      "content": "<|reserved_special_token_103|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128112": {
+      "content": "<|reserved_special_token_104|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128113": {
+      "content": "<|reserved_special_token_105|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128114": {
+      "content": "<|reserved_special_token_106|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128115": {
+      "content": "<|reserved_special_token_107|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128116": {
+      "content": "<|reserved_special_token_108|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128117": {
+      "content": "<|reserved_special_token_109|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128118": {
+      "content": "<|reserved_special_token_110|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128119": {
+      "content": "<|reserved_special_token_111|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128120": {
+      "content": "<|reserved_special_token_112|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128121": {
+      "content": "<|reserved_special_token_113|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128122": {
+      "content": "<|reserved_special_token_114|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128123": {
+      "content": "<|reserved_special_token_115|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128124": {
+      "content": "<|reserved_special_token_116|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128125": {
+      "content": "<|reserved_special_token_117|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128126": {
+      "content": "<|reserved_special_token_118|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128127": {
+      "content": "<|reserved_special_token_119|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128128": {
+      "content": "<|reserved_special_token_120|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128129": {
+      "content": "<|reserved_special_token_121|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128130": {
+      "content": "<|reserved_special_token_122|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128131": {
+      "content": "<|reserved_special_token_123|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128132": {
+      "content": "<|reserved_special_token_124|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128133": {
+      "content": "<|reserved_special_token_125|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128134": {
+      "content": "<|reserved_special_token_126|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128135": {
+      "content": "<|reserved_special_token_127|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128136": {
+      "content": "<|reserved_special_token_128|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128137": {
+      "content": "<|reserved_special_token_129|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128138": {
+      "content": "<|reserved_special_token_130|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128139": {
+      "content": "<|reserved_special_token_131|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128140": {
+      "content": "<|reserved_special_token_132|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128141": {
+      "content": "<|reserved_special_token_133|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128142": {
+      "content": "<|reserved_special_token_134|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128143": {
+      "content": "<|reserved_special_token_135|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128144": {
+      "content": "<|reserved_special_token_136|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128145": {
+      "content": "<|reserved_special_token_137|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128146": {
+      "content": "<|reserved_special_token_138|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128147": {
+      "content": "<|reserved_special_token_139|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128148": {
+      "content": "<|reserved_special_token_140|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128149": {
+      "content": "<|reserved_special_token_141|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128150": {
+      "content": "<|reserved_special_token_142|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128151": {
+      "content": "<|reserved_special_token_143|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128152": {
+      "content": "<|reserved_special_token_144|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128153": {
+      "content": "<|reserved_special_token_145|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128154": {
+      "content": "<|reserved_special_token_146|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128155": {
+      "content": "<|reserved_special_token_147|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128156": {
+      "content": "<|reserved_special_token_148|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128157": {
+      "content": "<|reserved_special_token_149|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128158": {
+      "content": "<|reserved_special_token_150|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128159": {
+      "content": "<|reserved_special_token_151|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128160": {
+      "content": "<|reserved_special_token_152|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128161": {
+      "content": "<|reserved_special_token_153|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128162": {
+      "content": "<|reserved_special_token_154|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128163": {
+      "content": "<|reserved_special_token_155|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128164": {
+      "content": "<|reserved_special_token_156|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128165": {
+      "content": "<|reserved_special_token_157|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128166": {
+      "content": "<|reserved_special_token_158|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128167": {
+      "content": "<|reserved_special_token_159|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128168": {
+      "content": "<|reserved_special_token_160|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128169": {
+      "content": "<|reserved_special_token_161|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128170": {
+      "content": "<|reserved_special_token_162|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128171": {
+      "content": "<|reserved_special_token_163|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128172": {
+      "content": "<|reserved_special_token_164|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128173": {
+      "content": "<|reserved_special_token_165|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128174": {
+      "content": "<|reserved_special_token_166|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128175": {
+      "content": "<|reserved_special_token_167|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128176": {
+      "content": "<|reserved_special_token_168|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128177": {
+      "content": "<|reserved_special_token_169|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128178": {
+      "content": "<|reserved_special_token_170|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128179": {
+      "content": "<|reserved_special_token_171|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128180": {
+      "content": "<|reserved_special_token_172|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128181": {
+      "content": "<|reserved_special_token_173|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128182": {
+      "content": "<|reserved_special_token_174|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128183": {
+      "content": "<|reserved_special_token_175|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128184": {
+      "content": "<|reserved_special_token_176|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128185": {
+      "content": "<|reserved_special_token_177|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128186": {
+      "content": "<|reserved_special_token_178|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128187": {
+      "content": "<|reserved_special_token_179|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128188": {
+      "content": "<|reserved_special_token_180|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128189": {
+      "content": "<|reserved_special_token_181|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128190": {
+      "content": "<|reserved_special_token_182|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128191": {
+      "content": "<|reserved_special_token_183|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128192": {
+      "content": "<|reserved_special_token_184|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128193": {
+      "content": "<|reserved_special_token_185|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128194": {
+      "content": "<|reserved_special_token_186|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128195": {
+      "content": "<|reserved_special_token_187|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128196": {
+      "content": "<|reserved_special_token_188|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128197": {
+      "content": "<|reserved_special_token_189|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128198": {
+      "content": "<|reserved_special_token_190|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128199": {
+      "content": "<|reserved_special_token_191|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128200": {
+      "content": "<|reserved_special_token_192|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128201": {
+      "content": "<|reserved_special_token_193|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128202": {
+      "content": "<|reserved_special_token_194|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128203": {
+      "content": "<|reserved_special_token_195|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128204": {
+      "content": "<|reserved_special_token_196|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128205": {
+      "content": "<|reserved_special_token_197|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128206": {
+      "content": "<|reserved_special_token_198|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128207": {
+      "content": "<|reserved_special_token_199|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128208": {
+      "content": "<|reserved_special_token_200|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128209": {
+      "content": "<|reserved_special_token_201|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128210": {
+      "content": "<|reserved_special_token_202|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128211": {
+      "content": "<|reserved_special_token_203|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128212": {
+      "content": "<|reserved_special_token_204|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128213": {
+      "content": "<|reserved_special_token_205|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128214": {
+      "content": "<|reserved_special_token_206|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128215": {
+      "content": "<|reserved_special_token_207|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128216": {
+      "content": "<|reserved_special_token_208|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128217": {
+      "content": "<|reserved_special_token_209|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128218": {
+      "content": "<|reserved_special_token_210|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128219": {
+      "content": "<|reserved_special_token_211|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128220": {
+      "content": "<|reserved_special_token_212|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128221": {
+      "content": "<|reserved_special_token_213|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128222": {
+      "content": "<|reserved_special_token_214|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128223": {
+      "content": "<|reserved_special_token_215|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128224": {
+      "content": "<|reserved_special_token_216|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128225": {
+      "content": "<|reserved_special_token_217|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128226": {
+      "content": "<|reserved_special_token_218|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128227": {
+      "content": "<|reserved_special_token_219|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128228": {
+      "content": "<|reserved_special_token_220|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128229": {
+      "content": "<|reserved_special_token_221|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128230": {
+      "content": "<|reserved_special_token_222|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128231": {
+      "content": "<|reserved_special_token_223|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128232": {
+      "content": "<|reserved_special_token_224|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128233": {
+      "content": "<|reserved_special_token_225|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128234": {
+      "content": "<|reserved_special_token_226|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128235": {
+      "content": "<|reserved_special_token_227|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128236": {
+      "content": "<|reserved_special_token_228|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128237": {
+      "content": "<|reserved_special_token_229|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128238": {
+      "content": "<|reserved_special_token_230|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128239": {
+      "content": "<|reserved_special_token_231|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128240": {
+      "content": "<|reserved_special_token_232|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128241": {
+      "content": "<|reserved_special_token_233|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128242": {
+      "content": "<|reserved_special_token_234|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128243": {
+      "content": "<|reserved_special_token_235|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128244": {
+      "content": "<|reserved_special_token_236|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128245": {
+      "content": "<|reserved_special_token_237|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128246": {
+      "content": "<|reserved_special_token_238|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128247": {
+      "content": "<|reserved_special_token_239|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128248": {
+      "content": "<|reserved_special_token_240|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128249": {
+      "content": "<|reserved_special_token_241|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128250": {
+      "content": "<|reserved_special_token_242|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128251": {
+      "content": "<|reserved_special_token_243|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128252": {
+      "content": "<|reserved_special_token_244|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128253": {
+      "content": "<|reserved_special_token_245|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128254": {
+      "content": "<|reserved_special_token_246|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128255": {
+      "content": "<|reserved_special_token_247|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|begin_of_text|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|end_of_text|>",
+  "extra_special_tokens": {},
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "pad_token": "<|reserved_special_token_247|>",
+  "tokenizer_class": "PreTrainedTokenizer"
+}

hotpotqa_test_knowledge_50_base/0_128_e3_3e-5/train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "total_flos": 1.8970279091854377e+18,
+    "train_loss": 0.46635930327050884,
+    "train_runtime": 1484.7838,
+    "train_samples": 14795,
+    "train_samples_per_second": 29.893,
+    "train_steps_per_second": 0.935
+}

hotpotqa_test_knowledge_50_base/0_128_e3_3e-5/trainer_state.json

@@ -0,0 +1,1982 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 500,
+  "global_step": 1389,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.010810810810810811,
+      "grad_norm": 0.6749756932258606,
+      "learning_rate": 1.7142857142857143e-06,
+      "loss": 1.6002,
+      "step": 5
+    },
+    {
+      "epoch": 0.021621621621621623,
+      "grad_norm": 0.6739484667778015,
+      "learning_rate": 3.857142857142857e-06,
+      "loss": 1.6545,
+      "step": 10
+    },
+    {
+      "epoch": 0.032432432432432434,
+      "grad_norm": 0.5106654167175293,
+      "learning_rate": 6e-06,
+      "loss": 1.5524,
+      "step": 15
+    },
+    {
+      "epoch": 0.043243243243243246,
+      "grad_norm": 0.5350106954574585,
+      "learning_rate": 8.142857142857142e-06,
+      "loss": 1.627,
+      "step": 20
+    },
+    {
+      "epoch": 0.05405405405405406,
+      "grad_norm": 0.4918515980243683,
+      "learning_rate": 1.0285714285714286e-05,
+      "loss": 1.5917,
+      "step": 25
+    },
+    {
+      "epoch": 0.06486486486486487,
+      "grad_norm": 0.5422373414039612,
+      "learning_rate": 1.242857142857143e-05,
+      "loss": 1.5633,
+      "step": 30
+    },
+    {
+      "epoch": 0.07567567567567568,
+      "grad_norm": 0.5008731484413147,
+      "learning_rate": 1.4571428571428571e-05,
+      "loss": 1.5382,
+      "step": 35
+    },
+    {
+      "epoch": 0.08648648648648649,
+      "grad_norm": 0.452938973903656,
+      "learning_rate": 1.6714285714285716e-05,
+      "loss": 1.5515,
+      "step": 40
+    },
+    {
+      "epoch": 0.0972972972972973,
+      "grad_norm": 0.49676385521888733,
+      "learning_rate": 1.8857142857142856e-05,
+      "loss": 1.5271,
+      "step": 45
+    },
+    {
+      "epoch": 0.10810810810810811,
+      "grad_norm": 0.5166845321655273,
+      "learning_rate": 2.1e-05,
+      "loss": 1.5582,
+      "step": 50
+    },
+    {
+      "epoch": 0.11891891891891893,
+      "grad_norm": 0.4737708270549774,
+      "learning_rate": 2.3142857142857145e-05,
+      "loss": 1.454,
+      "step": 55
+    },
+    {
+      "epoch": 0.12972972972972974,
+      "grad_norm": 0.5474684834480286,
+      "learning_rate": 2.5285714285714285e-05,
+      "loss": 1.418,
+      "step": 60
+    },
+    {
+      "epoch": 0.14054054054054055,
+      "grad_norm": 0.5287275314331055,
+      "learning_rate": 2.7428571428571428e-05,
+      "loss": 1.4695,
+      "step": 65
+    },
+    {
+      "epoch": 0.15135135135135136,
+      "grad_norm": 0.6109984517097473,
+      "learning_rate": 2.9571428571428575e-05,
+      "loss": 1.431,
+      "step": 70
+    },
+    {
+      "epoch": 0.16216216216216217,
+      "grad_norm": 0.6882091164588928,
+      "learning_rate": 2.999931924927058e-05,
+      "loss": 1.3871,
+      "step": 75
+    },
+    {
+      "epoch": 0.17297297297297298,
+      "grad_norm": 0.6718704104423523,
+      "learning_rate": 2.999655380533103e-05,
+      "loss": 1.4817,
+      "step": 80
+    },
+    {
+      "epoch": 0.1837837837837838,
+      "grad_norm": 0.6657449007034302,
+      "learning_rate": 2.999166151316113e-05,
+      "loss": 1.4664,
+      "step": 85
+    },
+    {
+      "epoch": 0.1945945945945946,
+      "grad_norm": 0.6535046100616455,
+      "learning_rate": 2.9984643066597815e-05,
+      "loss": 1.3768,
+      "step": 90
+    },
+    {
+      "epoch": 0.20540540540540542,
+      "grad_norm": 0.6701520085334778,
+      "learning_rate": 2.997549946101448e-05,
+      "loss": 1.2797,
+      "step": 95
+    },
+    {
+      "epoch": 0.21621621621621623,
+      "grad_norm": 0.6286271214485168,
+      "learning_rate": 2.9964231993179804e-05,
+      "loss": 1.315,
+      "step": 100
+    },
+    {
+      "epoch": 0.22702702702702704,
+      "grad_norm": 0.6639678478240967,
+      "learning_rate": 2.9950842261073867e-05,
+      "loss": 1.3076,
+      "step": 105
+    },
+    {
+      "epoch": 0.23783783783783785,
+      "grad_norm": 0.6478069424629211,
+      "learning_rate": 2.9935332163661465e-05,
+      "loss": 1.2784,
+      "step": 110
+    },
+    {
+      "epoch": 0.24864864864864866,
+      "grad_norm": 0.653148353099823,
+      "learning_rate": 2.9917703900622867e-05,
+      "loss": 1.3221,
+      "step": 115
+    },
+    {
+      "epoch": 0.2594594594594595,
+      "grad_norm": 0.6923774480819702,
+      "learning_rate": 2.9897959972041778e-05,
+      "loss": 1.3407,
+      "step": 120
+    },
+    {
+      "epoch": 0.2702702702702703,
+      "grad_norm": 0.7054397463798523,
+      "learning_rate": 2.9876103178050833e-05,
+      "loss": 1.2499,
+      "step": 125
+    },
+    {
+      "epoch": 0.2810810810810811,
+      "grad_norm": 0.7252037525177002,
+      "learning_rate": 2.9852136618434435e-05,
+      "loss": 1.1914,
+      "step": 130
+    },
+    {
+      "epoch": 0.2918918918918919,
+      "grad_norm": 0.8995503187179565,
+      "learning_rate": 2.9826063692189152e-05,
+      "loss": 1.2278,
+      "step": 135
+    },
+    {
+      "epoch": 0.3027027027027027,
+      "grad_norm": 0.8531567454338074,
+      "learning_rate": 2.9797888097041656e-05,
+      "loss": 1.1881,
+      "step": 140
+    },
+    {
+      "epoch": 0.31351351351351353,
+      "grad_norm": 0.8225644826889038,
+      "learning_rate": 2.9767613828924316e-05,
+      "loss": 1.1099,
+      "step": 145
+    },
+    {
+      "epoch": 0.32432432432432434,
+      "grad_norm": 0.7384702563285828,
+      "learning_rate": 2.9735245181408466e-05,
+      "loss": 1.124,
+      "step": 150
+    },
+    {
+      "epoch": 0.33513513513513515,
+      "grad_norm": 0.7921214699745178,
+      "learning_rate": 2.9700786745095483e-05,
+      "loss": 1.1146,
+      "step": 155
+    },
+    {
+      "epoch": 0.34594594594594597,
+      "grad_norm": 0.8876392841339111,
+      "learning_rate": 2.966424340696577e-05,
+      "loss": 1.0549,
+      "step": 160
+    },
+    {
+      "epoch": 0.3567567567567568,
+      "grad_norm": 0.8177557587623596,
+      "learning_rate": 2.9625620349685617e-05,
+      "loss": 1.1194,
+      "step": 165
+    },
+    {
+      "epoch": 0.3675675675675676,
+      "grad_norm": 0.9204846620559692,
+      "learning_rate": 2.9584923050872218e-05,
+      "loss": 1.147,
+      "step": 170
+    },
+    {
+      "epoch": 0.3783783783783784,
+      "grad_norm": 0.962609052658081,
+      "learning_rate": 2.9542157282316822e-05,
+      "loss": 1.0603,
+      "step": 175
+    },
+    {
+      "epoch": 0.3891891891891892,
+      "grad_norm": 0.9691535234451294,
+      "learning_rate": 2.9497329109166154e-05,
+      "loss": 1.0567,
+      "step": 180
+    },
+    {
+      "epoch": 0.4,
+      "grad_norm": 0.9528350234031677,
+      "learning_rate": 2.9450444889062243e-05,
+      "loss": 1.119,
+      "step": 185
+    },
+    {
+      "epoch": 0.41081081081081083,
+      "grad_norm": 1.0783238410949707,
+      "learning_rate": 2.940151127124076e-05,
+      "loss": 1.0226,
+      "step": 190
+    },
+    {
+      "epoch": 0.42162162162162165,
+      "grad_norm": 0.9221756458282471,
+      "learning_rate": 2.935053519558802e-05,
+      "loss": 1.0075,
+      "step": 195
+    },
+    {
+      "epoch": 0.43243243243243246,
+      "grad_norm": 0.9663214683532715,
+      "learning_rate": 2.929752389165673e-05,
+      "loss": 1.0586,
+      "step": 200
+    },
+    {
+      "epoch": 0.44324324324324327,
+      "grad_norm": 1.1029077768325806,
+      "learning_rate": 2.9242484877640706e-05,
+      "loss": 0.969,
+      "step": 205
+    },
+    {
+      "epoch": 0.4540540540540541,
+      "grad_norm": 0.9367722272872925,
+      "learning_rate": 2.9185425959308596e-05,
+      "loss": 0.9634,
+      "step": 210
+    },
+    {
+      "epoch": 0.4648648648648649,
+      "grad_norm": 1.0548895597457886,
+      "learning_rate": 2.912635522889686e-05,
+      "loss": 1.0103,
+      "step": 215
+    },
+    {
+      "epoch": 0.4756756756756757,
+      "grad_norm": 1.347251534461975,
+      "learning_rate": 2.9065281063962106e-05,
+      "loss": 0.94,
+      "step": 220
+    },
+    {
+      "epoch": 0.4864864864864865,
+      "grad_norm": 1.0111850500106812,
+      "learning_rate": 2.900221212619297e-05,
+      "loss": 0.9342,
+      "step": 225
+    },
+    {
+      "epoch": 0.4972972972972973,
+      "grad_norm": 0.9622715711593628,
+      "learning_rate": 2.893715736018168e-05,
+      "loss": 1.0053,
+      "step": 230
+    },
+    {
+      "epoch": 0.5081081081081081,
+      "grad_norm": 1.0898102521896362,
+      "learning_rate": 2.8870125992155527e-05,
+      "loss": 0.9361,
+      "step": 235
+    },
+    {
+      "epoch": 0.518918918918919,
+      "grad_norm": 1.1396633386611938,
+      "learning_rate": 2.8801127528668372e-05,
+      "loss": 0.921,
+      "step": 240
+    },
+    {
+      "epoch": 0.5297297297297298,
+      "grad_norm": 0.9959793090820312,
+      "learning_rate": 2.87301717552524e-05,
+      "loss": 0.8875,
+      "step": 245
+    },
+    {
+      "epoch": 0.5405405405405406,
+      "grad_norm": 1.1547890901565552,
+      "learning_rate": 2.8657268735030316e-05,
+      "loss": 0.8682,
+      "step": 250
+    },
+    {
+      "epoch": 0.5513513513513514,
+      "grad_norm": 1.0546422004699707,
+      "learning_rate": 2.8582428807288182e-05,
+      "loss": 0.8818,
+      "step": 255
+    },
+    {
+      "epoch": 0.5621621621621622,
+      "grad_norm": 1.1594375371932983,
+      "learning_rate": 2.850566258600903e-05,
+      "loss": 0.8826,
+      "step": 260
+    },
+    {
+      "epoch": 0.572972972972973,
+      "grad_norm": 1.2605596780776978,
+      "learning_rate": 2.8426980958367618e-05,
+      "loss": 0.8348,
+      "step": 265
+    },
+    {
+      "epoch": 0.5837837837837838,
+      "grad_norm": 1.194010853767395,
+      "learning_rate": 2.8346395083186336e-05,
+      "loss": 0.8428,
+      "step": 270
+    },
+    {
+      "epoch": 0.5945945945945946,
+      "grad_norm": 1.0932097434997559,
+      "learning_rate": 2.8263916389352673e-05,
+      "loss": 0.7917,
+      "step": 275
+    },
+    {
+      "epoch": 0.6054054054054054,
+      "grad_norm": 1.2288398742675781,
+      "learning_rate": 2.8179556574198312e-05,
+      "loss": 0.8138,
+      "step": 280
+    },
+    {
+      "epoch": 0.6162162162162163,
+      "grad_norm": 1.1970932483673096,
+      "learning_rate": 2.80933276018402e-05,
+      "loss": 0.8412,
+      "step": 285
+    },
+    {
+      "epoch": 0.6270270270270271,
+      "grad_norm": 1.0771979093551636,
+      "learning_rate": 2.800524170148377e-05,
+      "loss": 0.7551,
+      "step": 290
+    },
+    {
+      "epoch": 0.6378378378378379,
+      "grad_norm": 1.191752552986145,
+      "learning_rate": 2.7915311365688562e-05,
+      "loss": 0.7675,
+      "step": 295
+    },
+    {
+      "epoch": 0.6486486486486487,
+      "grad_norm": 1.2224516868591309,
+      "learning_rate": 2.78235493485965e-05,
+      "loss": 0.7927,
+      "step": 300
+    },
+    {
+      "epoch": 0.6594594594594595,
+      "grad_norm": 1.2681128978729248,
+      "learning_rate": 2.7729968664123056e-05,
+      "loss": 0.765,
+      "step": 305
+    },
+    {
+      "epoch": 0.6702702702702703,
+      "grad_norm": 1.0764217376708984,
+      "learning_rate": 2.7634582584111605e-05,
+      "loss": 0.7519,
+      "step": 310
+    },
+    {
+      "epoch": 0.6810810810810811,
+      "grad_norm": 1.1778703927993774,
+      "learning_rate": 2.753740463645115e-05,
+      "loss": 0.7052,
+      "step": 315
+    },
+    {
+      "epoch": 0.6918918918918919,
+      "grad_norm": 1.1717065572738647,
+      "learning_rate": 2.7438448603157802e-05,
+      "loss": 0.7107,
+      "step": 320
+    },
+    {
+      "epoch": 0.7027027027027027,
+      "grad_norm": 1.1341768503189087,
+      "learning_rate": 2.7337728518420143e-05,
+      "loss": 0.7493,
+      "step": 325
+    },
+    {
+      "epoch": 0.7135135135135136,
+      "grad_norm": 1.397209644317627,
+      "learning_rate": 2.723525866660889e-05,
+      "loss": 0.7303,
+      "step": 330
+    },
+    {
+      "epoch": 0.7243243243243244,
+      "grad_norm": 1.1951305866241455,
+      "learning_rate": 2.713105358025104e-05,
+      "loss": 0.7071,
+      "step": 335
+    },
+    {
+      "epoch": 0.7351351351351352,
+      "grad_norm": 1.187578797340393,
+      "learning_rate": 2.7025128037968824e-05,
+      "loss": 0.66,
+      "step": 340
+    },
+    {
+      "epoch": 0.745945945945946,
+      "grad_norm": 1.2162693738937378,
+      "learning_rate": 2.6917497062383776e-05,
+      "loss": 0.7046,
+      "step": 345
+    },
+    {
+      "epoch": 0.7567567567567568,
+      "grad_norm": 1.1686859130859375,
+      "learning_rate": 2.680817591798619e-05,
+      "loss": 0.6775,
+      "step": 350
+    },
+    {
+      "epoch": 0.7675675675675676,
+      "grad_norm": 1.3530421257019043,
+      "learning_rate": 2.669718010897026e-05,
+      "loss": 0.6515,
+      "step": 355
+    },
+    {
+      "epoch": 0.7783783783783784,
+      "grad_norm": 1.2877717018127441,
+      "learning_rate": 2.6584525377035257e-05,
+      "loss": 0.633,
+      "step": 360
+    },
+    {
+      "epoch": 0.7891891891891892,
+      "grad_norm": 1.2025889158248901,
+      "learning_rate": 2.6470227699152975e-05,
+      "loss": 0.6563,
+      "step": 365
+    },
+    {
+      "epoch": 0.8,
+      "grad_norm": 1.236899971961975,
+      "learning_rate": 2.635430328530186e-05,
+      "loss": 0.6818,
+      "step": 370
+    },
+    {
+      "epoch": 0.8108108108108109,
+      "grad_norm": 1.1630672216415405,
+      "learning_rate": 2.6236768576168065e-05,
+      "loss": 0.6385,
+      "step": 375
+    },
+    {
+      "epoch": 0.8216216216216217,
+      "grad_norm": 1.5487016439437866,
+      "learning_rate": 2.611764024081378e-05,
+      "loss": 0.6651,
+      "step": 380
+    },
+    {
+      "epoch": 0.8324324324324325,
+      "grad_norm": 1.2410399913787842,
+      "learning_rate": 2.5996935174313215e-05,
+      "loss": 0.5957,
+      "step": 385
+    },
+    {
+      "epoch": 0.8432432432432433,
+      "grad_norm": 1.1955680847167969,
+      "learning_rate": 2.587467049535645e-05,
+      "loss": 0.5786,
+      "step": 390
+    },
+    {
+      "epoch": 0.8540540540540541,
+      "grad_norm": 1.2050449848175049,
+      "learning_rate": 2.575086354382167e-05,
+      "loss": 0.5923,
+      "step": 395
+    },
+    {
+      "epoch": 0.8648648648648649,
+      "grad_norm": 1.3376110792160034,
+      "learning_rate": 2.5625531878315956e-05,
+      "loss": 0.6241,
+      "step": 400
+    },
+    {
+      "epoch": 0.8756756756756757,
+      "grad_norm": 1.433042049407959,
+      "learning_rate": 2.5498693273685074e-05,
+      "loss": 0.5906,
+      "step": 405
+    },
+    {
+      "epoch": 0.8864864864864865,
+      "grad_norm": 1.1621979475021362,
+      "learning_rate": 2.5370365718492615e-05,
+      "loss": 0.5668,
+      "step": 410
+    },
+    {
+      "epoch": 0.8972972972972973,
+      "grad_norm": 1.1963423490524292,
+      "learning_rate": 2.5240567412468816e-05,
+      "loss": 0.5354,
+      "step": 415
+    },
+    {
+      "epoch": 0.9081081081081082,
+      "grad_norm": 1.2802984714508057,
+      "learning_rate": 2.5109316763929414e-05,
+      "loss": 0.6125,
+      "step": 420
+    },
+    {
+      "epoch": 0.918918918918919,
+      "grad_norm": 1.1878982782363892,
+      "learning_rate": 2.497663238716495e-05,
+      "loss": 0.5889,
+      "step": 425
+    },
+    {
+      "epoch": 0.9297297297297298,
+      "grad_norm": 1.6462504863739014,
+      "learning_rate": 2.4842533099800822e-05,
+      "loss": 0.5493,
+      "step": 430
+    },
+    {
+      "epoch": 0.9405405405405406,
+      "grad_norm": 1.3050243854522705,
+      "learning_rate": 2.4707037920128554e-05,
+      "loss": 0.5801,
+      "step": 435
+    },
+    {
+      "epoch": 0.9513513513513514,
+      "grad_norm": 1.1411267518997192,
+      "learning_rate": 2.4570166064408556e-05,
+      "loss": 0.6013,
+      "step": 440
+    },
+    {
+      "epoch": 0.9621621621621622,
+      "grad_norm": 1.378894567489624,
+      "learning_rate": 2.4431936944144818e-05,
+      "loss": 0.4858,
+      "step": 445
+    },
+    {
+      "epoch": 0.972972972972973,
+      "grad_norm": 1.3321566581726074,
+      "learning_rate": 2.4292370163331936e-05,
+      "loss": 0.525,
+      "step": 450
+    },
+    {
+      "epoch": 0.9837837837837838,
+      "grad_norm": 1.2378548383712769,
+      "learning_rate": 2.4151485515674837e-05,
+      "loss": 0.5759,
+      "step": 455
+    },
+    {
+      "epoch": 0.9945945945945946,
+      "grad_norm": 1.3089052438735962,
+      "learning_rate": 2.400930298178155e-05,
+      "loss": 0.5343,
+      "step": 460
+    },
+    {
+      "epoch": 1.0043243243243243,
+      "grad_norm": 1.2367912530899048,
+      "learning_rate": 2.3865842726329556e-05,
+      "loss": 0.5467,
+      "step": 465
+    },
+    {
+      "epoch": 1.0151351351351352,
+      "grad_norm": 1.1718990802764893,
+      "learning_rate": 2.3721125095205935e-05,
+      "loss": 0.4462,
+      "step": 470
+    },
+    {
+      "epoch": 1.025945945945946,
+      "grad_norm": 1.3019416332244873,
+      "learning_rate": 2.357517061262189e-05,
+      "loss": 0.4097,
+      "step": 475
+    },
+    {
+      "epoch": 1.0367567567567568,
+      "grad_norm": 1.4085646867752075,
+      "learning_rate": 2.3427999978201956e-05,
+      "loss": 0.5282,
+      "step": 480
+    },
+    {
+      "epoch": 1.0475675675675675,
+      "grad_norm": 1.1572306156158447,
+      "learning_rate": 2.3279634064048308e-05,
+      "loss": 0.4134,
+      "step": 485
+    },
+    {
+      "epoch": 1.0583783783783784,
+      "grad_norm": 1.244713306427002,
+      "learning_rate": 2.3130093911780642e-05,
+      "loss": 0.4335,
+      "step": 490
+    },
+    {
+      "epoch": 1.0691891891891891,
+      "grad_norm": 1.173665165901184,
+      "learning_rate": 2.2979400729552014e-05,
+      "loss": 0.444,
+      "step": 495
+    },
+    {
+      "epoch": 1.08,
+      "grad_norm": 1.407321810722351,
+      "learning_rate": 2.2827575889041007e-05,
+      "loss": 0.4337,
+      "step": 500
+    },
+    {
+      "epoch": 1.0908108108108108,
+      "grad_norm": 1.3847451210021973,
+      "learning_rate": 2.267464092242078e-05,
+      "loss": 0.4213,
+      "step": 505
+    },
+    {
+      "epoch": 1.1016216216216217,
+      "grad_norm": 1.4645717144012451,
+      "learning_rate": 2.2520617519305325e-05,
+      "loss": 0.4171,
+      "step": 510
+    },
+    {
+      "epoch": 1.1124324324324324,
+      "grad_norm": 1.3133924007415771,
+      "learning_rate": 2.2365527523673368e-05,
+      "loss": 0.4461,
+      "step": 515
+    },
+    {
+      "epoch": 1.1232432432432433,
+      "grad_norm": 1.4021127223968506,
+      "learning_rate": 2.2209392930770424e-05,
+      "loss": 0.4047,
+      "step": 520
+    },
+    {
+      "epoch": 1.134054054054054,
+      "grad_norm": 1.2070517539978027,
+      "learning_rate": 2.2052235883989355e-05,
+      "loss": 0.4069,
+      "step": 525
+    },
+    {
+      "epoch": 1.144864864864865,
+      "grad_norm": 1.3282095193862915,
+      "learning_rate": 2.189407867172997e-05,
+      "loss": 0.406,
+      "step": 530
+    },
+    {
+      "epoch": 1.1556756756756756,
+      "grad_norm": 1.159316062927246,
+      "learning_rate": 2.1734943724237986e-05,
+      "loss": 0.4207,
+      "step": 535
+    },
+    {
+      "epoch": 1.1664864864864866,
+      "grad_norm": 1.5306358337402344,
+      "learning_rate": 2.157485361042398e-05,
+      "loss": 0.388,
+      "step": 540
+    },
+    {
+      "epoch": 1.1772972972972973,
+      "grad_norm": 1.1564973592758179,
+      "learning_rate": 2.1413831034662536e-05,
+      "loss": 0.395,
+      "step": 545
+    },
+    {
+      "epoch": 1.1881081081081082,
+      "grad_norm": 1.439918875694275,
+      "learning_rate": 2.1251898833572303e-05,
+      "loss": 0.3825,
+      "step": 550
+    },
+    {
+      "epoch": 1.1989189189189189,
+      "grad_norm": 1.30429208278656,
+      "learning_rate": 2.1089079972777238e-05,
+      "loss": 0.3605,
+      "step": 555
+    },
+    {
+      "epoch": 1.2097297297297298,
+      "grad_norm": 1.2694649696350098,
+      "learning_rate": 2.092539754364957e-05,
+      "loss": 0.3555,
+      "step": 560
+    },
+    {
+      "epoch": 1.2205405405405405,
+      "grad_norm": 1.301515817642212,
+      "learning_rate": 2.07608747600349e-05,
+      "loss": 0.3516,
+      "step": 565
+    },
+    {
+      "epoch": 1.2313513513513514,
+      "grad_norm": 1.297397255897522,
+      "learning_rate": 2.0595534954959984e-05,
+      "loss": 0.3977,
+      "step": 570
+    },
+    {
+      "epoch": 1.2421621621621621,
+      "grad_norm": 1.3568118810653687,
+      "learning_rate": 2.0429401577323576e-05,
+      "loss": 0.3798,
+      "step": 575
+    },
+    {
+      "epoch": 1.252972972972973,
+      "grad_norm": 1.360713243484497,
+      "learning_rate": 2.0262498188570855e-05,
+      "loss": 0.3432,
+      "step": 580
+    },
+    {
+      "epoch": 1.2637837837837838,
+      "grad_norm": 1.4393284320831299,
+      "learning_rate": 2.0094848459351853e-05,
+      "loss": 0.34,
+      "step": 585
+    },
+    {
+      "epoch": 1.2745945945945947,
+      "grad_norm": 1.3435497283935547,
+      "learning_rate": 1.992647616616447e-05,
+      "loss": 0.3989,
+      "step": 590
+    },
+    {
+      "epoch": 1.2854054054054054,
+      "grad_norm": 1.312248945236206,
+      "learning_rate": 1.9757405187982397e-05,
+      "loss": 0.3532,
+      "step": 595
+    },
+    {
+      "epoch": 1.2962162162162163,
+      "grad_norm": 1.5269815921783447,
+      "learning_rate": 1.9587659502868546e-05,
+      "loss": 0.3891,
+      "step": 600
+    },
+    {
+      "epoch": 1.307027027027027,
+      "grad_norm": 1.4130357503890991,
+      "learning_rate": 1.9417263184574453e-05,
+      "loss": 0.3713,
+      "step": 605
+    },
+    {
+      "epoch": 1.3178378378378377,
+      "grad_norm": 1.3583375215530396,
+      "learning_rate": 1.9246240399126036e-05,
+      "loss": 0.3474,
+      "step": 610
+    },
+    {
+      "epoch": 1.3286486486486486,
+      "grad_norm": 1.3835967779159546,
+      "learning_rate": 1.907461540139633e-05,
+      "loss": 0.3497,
+      "step": 615
+    },
+    {
+      "epoch": 1.3394594594594595,
+      "grad_norm": 1.390000820159912,
+      "learning_rate": 1.8902412531665613e-05,
+      "loss": 0.3002,
+      "step": 620
+    },
+    {
+      "epoch": 1.3502702702702702,
+      "grad_norm": 1.2552826404571533,
+      "learning_rate": 1.872965621216938e-05,
+      "loss": 0.2847,
+      "step": 625
+    },
+    {
+      "epoch": 1.361081081081081,
+      "grad_norm": 1.3595759868621826,
+      "learning_rate": 1.855637094363474e-05,
+      "loss": 0.3211,
+      "step": 630
+    },
+    {
+      "epoch": 1.3718918918918919,
+      "grad_norm": 1.2867708206176758,
+      "learning_rate": 1.8382581301805676e-05,
+      "loss": 0.3547,
+      "step": 635
+    },
+    {
+      "epoch": 1.3827027027027028,
+      "grad_norm": 1.43692946434021,
+      "learning_rate": 1.8208311933957606e-05,
+      "loss": 0.3251,
+      "step": 640
+    },
+    {
+      "epoch": 1.3935135135135135,
+      "grad_norm": 1.396464467048645,
+      "learning_rate": 1.80335875554019e-05,
+      "loss": 0.2924,
+      "step": 645
+    },
+    {
+      "epoch": 1.4043243243243242,
+      "grad_norm": 1.2472106218338013,
+      "learning_rate": 1.7858432945980645e-05,
+      "loss": 0.3178,
+      "step": 650
+    },
+    {
+      "epoch": 1.4151351351351351,
+      "grad_norm": 1.4300161600112915,
+      "learning_rate": 1.7682872946552352e-05,
+      "loss": 0.3295,
+      "step": 655
+    },
+    {
+      "epoch": 1.425945945945946,
+      "grad_norm": 1.3491531610488892,
+      "learning_rate": 1.750693245546893e-05,
+      "loss": 0.3242,
+      "step": 660
+    },
+    {
+      "epoch": 1.4367567567567567,
+      "grad_norm": 1.2698180675506592,
+      "learning_rate": 1.7330636425044553e-05,
+      "loss": 0.2917,
+      "step": 665
+    },
+    {
+      "epoch": 1.4475675675675674,
+      "grad_norm": 1.3012919425964355,
+      "learning_rate": 1.7154009858016867e-05,
+      "loss": 0.3006,
+      "step": 670
+    },
+    {
+      "epoch": 1.4583783783783784,
+      "grad_norm": 1.333957314491272,
+      "learning_rate": 1.697707780400102e-05,
+      "loss": 0.2756,
+      "step": 675
+    },
+    {
+      "epoch": 1.4691891891891893,
+      "grad_norm": 1.436802625656128,
+      "learning_rate": 1.67998653559371e-05,
+      "loss": 0.3122,
+      "step": 680
+    },
+    {
+      "epoch": 1.48,
+      "grad_norm": 1.6837379932403564,
+      "learning_rate": 1.6622397646531352e-05,
+      "loss": 0.2826,
+      "step": 685
+    },
+    {
+      "epoch": 1.4908108108108107,
+      "grad_norm": 1.2060457468032837,
+      "learning_rate": 1.644469984469182e-05,
+      "loss": 0.2958,
+      "step": 690
+    },
+    {
+      "epoch": 1.5016216216216216,
+      "grad_norm": 1.5616896152496338,
+      "learning_rate": 1.6266797151958815e-05,
+      "loss": 0.3542,
+      "step": 695
+    },
+    {
+      "epoch": 1.5124324324324325,
+      "grad_norm": 1.284088134765625,
+      "learning_rate": 1.6088714798930806e-05,
+      "loss": 0.3139,
+      "step": 700
+    },
+    {
+      "epoch": 1.5232432432432432,
+      "grad_norm": 1.1483250856399536,
+      "learning_rate": 1.59104780416861e-05,
+      "loss": 0.2875,
+      "step": 705
+    },
+    {
+      "epoch": 1.534054054054054,
+      "grad_norm": 1.4885141849517822,
+      "learning_rate": 1.5732112158200995e-05,
+      "loss": 0.2942,
+      "step": 710
+    },
+    {
+      "epoch": 1.5448648648648649,
+      "grad_norm": 1.228744626045227,
+      "learning_rate": 1.5553642444764808e-05,
+      "loss": 0.2939,
+      "step": 715
+    },
+    {
+      "epoch": 1.5556756756756758,
+      "grad_norm": 1.2387549877166748,
+      "learning_rate": 1.5375094212392253e-05,
+      "loss": 0.2882,
+      "step": 720
+    },
+    {
+      "epoch": 1.5664864864864865,
+      "grad_norm": 1.2862504720687866,
+      "learning_rate": 1.5196492783233815e-05,
+      "loss": 0.2975,
+      "step": 725
+    },
+    {
+      "epoch": 1.5772972972972972,
+      "grad_norm": 1.3139840364456177,
+      "learning_rate": 1.501786348698446e-05,
+      "loss": 0.2802,
+      "step": 730
+    },
+    {
+      "epoch": 1.588108108108108,
+      "grad_norm": 1.3352205753326416,
+      "learning_rate": 1.4839231657291343e-05,
+      "loss": 0.2577,
+      "step": 735
+    },
+    {
+      "epoch": 1.598918918918919,
+      "grad_norm": 1.234085202217102,
+      "learning_rate": 1.4660622628160921e-05,
+      "loss": 0.2855,
+      "step": 740
+    },
+    {
+      "epoch": 1.6097297297297297,
+      "grad_norm": 1.0972801446914673,
+      "learning_rate": 1.4482061730366009e-05,
+      "loss": 0.2314,
+      "step": 745
+    },
+    {
+      "epoch": 1.6205405405405404,
+      "grad_norm": 1.3880709409713745,
+      "learning_rate": 1.4303574287853324e-05,
+      "loss": 0.2443,
+      "step": 750
+    },
+    {
+      "epoch": 1.6313513513513513,
+      "grad_norm": 1.198792576789856,
+      "learning_rate": 1.4125185614151967e-05,
+      "loss": 0.217,
+      "step": 755
+    },
+    {
+      "epoch": 1.6421621621621623,
+      "grad_norm": 1.3121585845947266,
+      "learning_rate": 1.3946921008783418e-05,
+      "loss": 0.2606,
+      "step": 760
+    },
+    {
+      "epoch": 1.652972972972973,
+      "grad_norm": 1.3656197786331177,
+      "learning_rate": 1.3768805753673465e-05,
+      "loss": 0.2372,
+      "step": 765
+    },
+    {
+      "epoch": 1.6637837837837837,
+      "grad_norm": 1.1282933950424194,
+      "learning_rate": 1.359086510956668e-05,
+      "loss": 0.2716,
+      "step": 770
+    },
+    {
+      "epoch": 1.6745945945945946,
+      "grad_norm": 1.2672842741012573,
+      "learning_rate": 1.3413124312443874e-05,
+      "loss": 0.2448,
+      "step": 775
+    },
+    {
+      "epoch": 1.6854054054054055,
+      "grad_norm": 1.3546538352966309,
+      "learning_rate": 1.3235608569943059e-05,
+      "loss": 0.2314,
+      "step": 780
+    },
+    {
+      "epoch": 1.6962162162162162,
+      "grad_norm": 1.421842336654663,
+      "learning_rate": 1.3058343057784458e-05,
+      "loss": 0.2107,
+      "step": 785
+    },
+    {
+      "epoch": 1.707027027027027,
+      "grad_norm": 1.2018187046051025,
+      "learning_rate": 1.2881352916199988e-05,
+      "loss": 0.2513,
+      "step": 790
+    },
+    {
+      "epoch": 1.7178378378378378,
+      "grad_norm": 1.2349228858947754,
+      "learning_rate": 1.270466324636786e-05,
+      "loss": 0.2437,
+      "step": 795
+    },
+    {
+      "epoch": 1.7286486486486488,
+      "grad_norm": 1.2098289728164673,
+      "learning_rate": 1.252829910685263e-05,
+      "loss": 0.2504,
+      "step": 800
+    },
+    {
+      "epoch": 1.7394594594594595,
+      "grad_norm": 1.5244630575180054,
+      "learning_rate": 1.2352285510051324e-05,
+      "loss": 0.2744,
+      "step": 805
+    },
+    {
+      "epoch": 1.7502702702702702,
+      "grad_norm": 1.1220543384552002,
+      "learning_rate": 1.2176647418646156e-05,
+      "loss": 0.27,
+      "step": 810
+    },
+    {
+      "epoch": 1.761081081081081,
+      "grad_norm": 1.3971467018127441,
+      "learning_rate": 1.2001409742064245e-05,
+      "loss": 0.2063,
+      "step": 815
+    },
+    {
+      "epoch": 1.771891891891892,
+      "grad_norm": 1.285480260848999,
+      "learning_rate": 1.1826597332944873e-05,
+      "loss": 0.2465,
+      "step": 820
+    },
+    {
+      "epoch": 1.7827027027027027,
+      "grad_norm": 1.1862159967422485,
+      "learning_rate": 1.1652234983614848e-05,
+      "loss": 0.1919,
+      "step": 825
+    },
+    {
+      "epoch": 1.7935135135135134,
+      "grad_norm": 1.2218137979507446,
+      "learning_rate": 1.1478347422572396e-05,
+      "loss": 0.2234,
+      "step": 830
+    },
+    {
+      "epoch": 1.8043243243243243,
+      "grad_norm": 1.3611098527908325,
+      "learning_rate": 1.1304959310980097e-05,
+      "loss": 0.242,
+      "step": 835
+    },
+    {
+      "epoch": 1.8151351351351352,
+      "grad_norm": 1.190139889717102,
+      "learning_rate": 1.113209523916737e-05,
+      "loss": 0.2234,
+      "step": 840
+    },
+    {
+      "epoch": 1.825945945945946,
+      "grad_norm": 1.226676344871521,
+      "learning_rate": 1.0959779723143024e-05,
+      "loss": 0.2093,
+      "step": 845
+    },
+    {
+      "epoch": 1.8367567567567566,
+      "grad_norm": 1.5197325944900513,
+      "learning_rate": 1.0788037201118346e-05,
+      "loss": 0.1876,
+      "step": 850
+    },
+    {
+      "epoch": 1.8475675675675676,
+      "grad_norm": 1.2957510948181152,
+      "learning_rate": 1.0616892030041184e-05,
+      "loss": 0.1993,
+      "step": 855
+    },
+    {
+      "epoch": 1.8583783783783785,
+      "grad_norm": 1.3621931076049805,
+      "learning_rate": 1.0446368482141612e-05,
+      "loss": 0.2042,
+      "step": 860
+    },
+    {
+      "epoch": 1.8691891891891892,
+      "grad_norm": 1.3100106716156006,
+      "learning_rate": 1.027649074148956e-05,
+      "loss": 0.2088,
+      "step": 865
+    },
+    {
+      "epoch": 1.88,
+      "grad_norm": 1.2206536531448364,
+      "learning_rate": 1.0107282900565009e-05,
+      "loss": 0.1825,
+      "step": 870
+    },
+    {
+      "epoch": 1.8908108108108108,
+      "grad_norm": 1.2510806322097778,
+      "learning_rate": 9.938768956841077e-06,
+      "loss": 0.1935,
+      "step": 875
+    },
+    {
+      "epoch": 1.9016216216216217,
+      "grad_norm": 1.1333528757095337,
+      "learning_rate": 9.770972809380696e-06,
+      "loss": 0.2062,
+      "step": 880
+    },
+    {
+      "epoch": 1.9124324324324324,
+      "grad_norm": 1.1528996229171753,
+      "learning_rate": 9.603918255447141e-06,
+      "loss": 0.1802,
+      "step": 885
+    },
+    {
+      "epoch": 1.9232432432432431,
+      "grad_norm": 1.113772988319397,
+      "learning_rate": 9.437628987129084e-06,
+      "loss": 0.1865,
+      "step": 890
+    },
+    {
+      "epoch": 1.934054054054054,
+      "grad_norm": 1.3337033987045288,
+      "learning_rate": 9.272128587980498e-06,
+      "loss": 0.2258,
+      "step": 895
+    },
+    {
+      "epoch": 1.944864864864865,
+      "grad_norm": 1.5289134979248047,
+      "learning_rate": 9.107440529675971e-06,
+      "loss": 0.1924,
+      "step": 900
+    },
+    {
+      "epoch": 1.9556756756756757,
+      "grad_norm": 1.144655704498291,
+      "learning_rate": 8.943588168681937e-06,
+      "loss": 0.1913,
+      "step": 905
+    },
+    {
+      "epoch": 1.9664864864864864,
+      "grad_norm": 1.1703957319259644,
+      "learning_rate": 8.780594742944159e-06,
+      "loss": 0.1671,
+      "step": 910
+    },
+    {
+      "epoch": 1.9772972972972973,
+      "grad_norm": 1.2245736122131348,
+      "learning_rate": 8.618483368592088e-06,
+      "loss": 0.1922,
+      "step": 915
+    },
+    {
+      "epoch": 1.9881081081081082,
+      "grad_norm": 1.141701102256775,
+      "learning_rate": 8.457277036660463e-06,
+      "loss": 0.1909,
+      "step": 920
+    },
+    {
+      "epoch": 1.998918918918919,
+      "grad_norm": 1.3344765901565552,
+      "learning_rate": 8.296998609828704e-06,
+      "loss": 0.1666,
+      "step": 925
+    },
+    {
+      "epoch": 2.0086486486486486,
+      "grad_norm": 1.3570910692214966,
+      "learning_rate": 8.13767081917841e-06,
+      "loss": 0.1632,
+      "step": 930
+    },
+    {
+      "epoch": 2.0194594594594593,
+      "grad_norm": 1.010612964630127,
+      "learning_rate": 7.97931626096964e-06,
+      "loss": 0.1403,
+      "step": 935
+    },
+    {
+      "epoch": 2.0302702702702704,
+      "grad_norm": 1.2923632860183716,
+      "learning_rate": 7.821957393436202e-06,
+      "loss": 0.1433,
+      "step": 940
+    },
+    {
+      "epoch": 2.041081081081081,
+      "grad_norm": 1.1319085359573364,
+      "learning_rate": 7.665616533600628e-06,
+      "loss": 0.15,
+      "step": 945
+    },
+    {
+      "epoch": 2.051891891891892,
+      "grad_norm": 1.172631025314331,
+      "learning_rate": 7.5103158541090665e-06,
+      "loss": 0.1367,
+      "step": 950
+    },
+    {
+      "epoch": 2.0627027027027025,
+      "grad_norm": 1.079500675201416,
+      "learning_rate": 7.356077380086726e-06,
+      "loss": 0.1169,
+      "step": 955
+    },
+    {
+      "epoch": 2.0735135135135137,
+      "grad_norm": 1.3374136686325073,
+      "learning_rate": 7.202922986014228e-06,
+      "loss": 0.1247,
+      "step": 960
+    },
+    {
+      "epoch": 2.0843243243243244,
+      "grad_norm": 1.0975528955459595,
+      "learning_rate": 7.050874392625302e-06,
+      "loss": 0.1366,
+      "step": 965
+    },
+    {
+      "epoch": 2.095135135135135,
+      "grad_norm": 1.2024213075637817,
+      "learning_rate": 6.899953163826292e-06,
+      "loss": 0.1312,
+      "step": 970
+    },
+    {
+      "epoch": 2.1059459459459458,
+      "grad_norm": 1.082950472831726,
+      "learning_rate": 6.7501807036379125e-06,
+      "loss": 0.1357,
+      "step": 975
+    },
+    {
+      "epoch": 2.116756756756757,
+      "grad_norm": 1.2047678232192993,
+      "learning_rate": 6.601578253159698e-06,
+      "loss": 0.151,
+      "step": 980
+    },
+    {
+      "epoch": 2.1275675675675676,
+      "grad_norm": 1.2053114175796509,
+      "learning_rate": 6.454166887557508e-06,
+      "loss": 0.1277,
+      "step": 985
+    },
+    {
+      "epoch": 2.1383783783783783,
+      "grad_norm": 1.0474656820297241,
+      "learning_rate": 6.307967513074605e-06,
+      "loss": 0.1319,
+      "step": 990
+    },
+    {
+      "epoch": 2.149189189189189,
+      "grad_norm": 1.2605841159820557,
+      "learning_rate": 6.163000864066698e-06,
+      "loss": 0.1478,
+      "step": 995
+    },
+    {
+      "epoch": 2.16,
+      "grad_norm": 0.9077703952789307,
+      "learning_rate": 6.019287500061326e-06,
+      "loss": 0.1534,
+      "step": 1000
+    },
+    {
+      "epoch": 2.170810810810811,
+      "grad_norm": 1.0507147312164307,
+      "learning_rate": 5.876847802842052e-06,
+      "loss": 0.1381,
+      "step": 1005
+    },
+    {
+      "epoch": 2.1816216216216215,
+      "grad_norm": 1.0697144269943237,
+      "learning_rate": 5.73570197355788e-06,
+      "loss": 0.1324,
+      "step": 1010
+    },
+    {
+      "epoch": 2.1924324324324322,
+      "grad_norm": 1.0238703489303589,
+      "learning_rate": 5.595870029858268e-06,
+      "loss": 0.1278,
+      "step": 1015
+    },
+    {
+      "epoch": 2.2032432432432434,
+      "grad_norm": 1.0087915658950806,
+      "learning_rate": 5.4573718030541965e-06,
+      "loss": 0.1288,
+      "step": 1020
+    },
+    {
+      "epoch": 2.214054054054054,
+      "grad_norm": 0.9904815554618835,
+      "learning_rate": 5.320226935305609e-06,
+      "loss": 0.1173,
+      "step": 1025
+    },
+    {
+      "epoch": 2.224864864864865,
+      "grad_norm": 1.0402568578720093,
+      "learning_rate": 5.184454876835746e-06,
+      "loss": 0.1243,
+      "step": 1030
+    },
+    {
+      "epoch": 2.2356756756756755,
+      "grad_norm": 1.169760823249817,
+      "learning_rate": 5.05007488317265e-06,
+      "loss": 0.1212,
+      "step": 1035
+    },
+    {
+      "epoch": 2.2464864864864866,
+      "grad_norm": 1.160381555557251,
+      "learning_rate": 4.917106012418294e-06,
+      "loss": 0.1327,
+      "step": 1040
+    },
+    {
+      "epoch": 2.2572972972972973,
+      "grad_norm": 0.8716002106666565,
+      "learning_rate": 4.7855671225457225e-06,
+      "loss": 0.1173,
+      "step": 1045
+    },
+    {
+      "epoch": 2.268108108108108,
+      "grad_norm": 0.8988707065582275,
+      "learning_rate": 4.655476868724566e-06,
+      "loss": 0.1179,
+      "step": 1050
+    },
+    {
+      "epoch": 2.2789189189189187,
+      "grad_norm": 1.1103434562683105,
+      "learning_rate": 4.526853700675325e-06,
+      "loss": 0.1284,
+      "step": 1055
+    },
+    {
+      "epoch": 2.28972972972973,
+      "grad_norm": 1.0457454919815063,
+      "learning_rate": 4.399715860052781e-06,
+      "loss": 0.1293,
+      "step": 1060
+    },
+    {
+      "epoch": 2.3005405405405406,
+      "grad_norm": 0.9258773326873779,
+      "learning_rate": 4.274081377858909e-06,
+      "loss": 0.0914,
+      "step": 1065
+    },
+    {
+      "epoch": 2.3113513513513513,
+      "grad_norm": 1.0390188694000244,
+      "learning_rate": 4.149968071885682e-06,
+      "loss": 0.1125,
+      "step": 1070
+    },
+    {
+      "epoch": 2.322162162162162,
+      "grad_norm": 1.0505141019821167,
+      "learning_rate": 4.027393544188129e-06,
+      "loss": 0.122,
+      "step": 1075
+    },
+    {
+      "epoch": 2.332972972972973,
+      "grad_norm": 1.2161805629730225,
+      "learning_rate": 3.9063751785879365e-06,
+      "loss": 0.109,
+      "step": 1080
+    },
+    {
+      "epoch": 2.343783783783784,
+      "grad_norm": 1.097092628479004,
+      "learning_rate": 3.786930138208046e-06,
+      "loss": 0.1149,
+      "step": 1085
+    },
+    {
+      "epoch": 2.3545945945945945,
+      "grad_norm": 0.751530647277832,
+      "learning_rate": 3.6690753630385436e-06,
+      "loss": 0.1093,
+      "step": 1090
+    },
+    {
+      "epoch": 2.3654054054054052,
+      "grad_norm": 0.966195821762085,
+      "learning_rate": 3.552827567534186e-06,
+      "loss": 0.1116,
+      "step": 1095
+    },
+    {
+      "epoch": 2.3762162162162164,
+      "grad_norm": 1.1819593906402588,
+      "learning_rate": 3.4382032382439036e-06,
+      "loss": 0.1167,
+      "step": 1100
+    },
+    {
+      "epoch": 2.387027027027027,
+      "grad_norm": 0.9405239224433899,
+      "learning_rate": 3.3252186314726447e-06,
+      "loss": 0.1174,
+      "step": 1105
+    },
+    {
+      "epoch": 2.3978378378378378,
+      "grad_norm": 1.1678102016448975,
+      "learning_rate": 3.213889770975881e-06,
+      "loss": 0.1196,
+      "step": 1110
+    },
+    {
+      "epoch": 2.4086486486486485,
+      "grad_norm": 0.9688264727592468,
+      "learning_rate": 3.1042324456870514e-06,
+      "loss": 0.119,
+      "step": 1115
+    },
+    {
+      "epoch": 2.4194594594594596,
+      "grad_norm": 0.8630348443984985,
+      "learning_rate": 2.996262207478356e-06,
+      "loss": 0.1265,
+      "step": 1120
+    },
+    {
+      "epoch": 2.4302702702702703,
+      "grad_norm": 0.9478833079338074,
+      "learning_rate": 2.8899943689551493e-06,
+      "loss": 0.122,
+      "step": 1125
+    },
+    {
+      "epoch": 2.441081081081081,
+      "grad_norm": 0.9283960461616516,
+      "learning_rate": 2.7854440012842684e-06,
+      "loss": 0.1087,
+      "step": 1130
+    },
+    {
+      "epoch": 2.4518918918918917,
+      "grad_norm": 0.8862292766571045,
+      "learning_rate": 2.6826259320565903e-06,
+      "loss": 0.1025,
+      "step": 1135
+    },
+    {
+      "epoch": 2.462702702702703,
+      "grad_norm": 1.147686243057251,
+      "learning_rate": 2.581554743184158e-06,
+      "loss": 0.1315,
+      "step": 1140
+    },
+    {
+      "epoch": 2.4735135135135136,
+      "grad_norm": 0.9898723363876343,
+      "learning_rate": 2.482244768832146e-06,
+      "loss": 0.1242,
+      "step": 1145
+    },
+    {
+      "epoch": 2.4843243243243243,
+      "grad_norm": 1.0704761743545532,
+      "learning_rate": 2.384710093385929e-06,
+      "loss": 0.115,
+      "step": 1150
+    },
+    {
+      "epoch": 2.495135135135135,
+      "grad_norm": 0.9278268814086914,
+      "learning_rate": 2.288964549453633e-06,
+      "loss": 0.1071,
+      "step": 1155
+    },
+    {
+      "epoch": 2.505945945945946,
+      "grad_norm": 0.8467145562171936,
+      "learning_rate": 2.1950217159043197e-06,
+      "loss": 0.114,
+      "step": 1160
+    },
+    {
+      "epoch": 2.516756756756757,
+      "grad_norm": 1.036379337310791,
+      "learning_rate": 2.102894915942243e-06,
+      "loss": 0.1097,
+      "step": 1165
+    },
+    {
+      "epoch": 2.5275675675675675,
+      "grad_norm": 0.9519475102424622,
+      "learning_rate": 2.0125972152172777e-06,
+      "loss": 0.1218,
+      "step": 1170
+    },
+    {
+      "epoch": 2.538378378378378,
+      "grad_norm": 0.8463801741600037,
+      "learning_rate": 1.924141419971949e-06,
+      "loss": 0.0968,
+      "step": 1175
+    },
+    {
+      "epoch": 2.5491891891891894,
+      "grad_norm": 0.900498628616333,
+      "learning_rate": 1.8375400752251943e-06,
+      "loss": 0.1054,
+      "step": 1180
+    },
+    {
+      "epoch": 2.56,
+      "grad_norm": 0.8663133382797241,
+      "learning_rate": 1.75280546299322e-06,
+      "loss": 0.0968,
+      "step": 1185
+    },
+    {
+      "epoch": 2.5708108108108108,
+      "grad_norm": 0.8068374395370483,
+      "learning_rate": 1.6699496005476244e-06,
+      "loss": 0.099,
+      "step": 1190
+    },
+    {
+      "epoch": 2.581621621621622,
+      "grad_norm": 0.7136590480804443,
+      "learning_rate": 1.5889842387110742e-06,
+      "loss": 0.1047,
+      "step": 1195
+    },
+    {
+      "epoch": 2.5924324324324326,
+      "grad_norm": 0.948045551776886,
+      "learning_rate": 1.5099208601907777e-06,
+      "loss": 0.1039,
+      "step": 1200
+    },
+    {
+      "epoch": 2.6032432432432433,
+      "grad_norm": 0.7788717150688171,
+      "learning_rate": 1.4327706779499927e-06,
+      "loss": 0.0927,
+      "step": 1205
+    },
+    {
+      "epoch": 2.614054054054054,
+      "grad_norm": 0.8725752830505371,
+      "learning_rate": 1.3575446336177506e-06,
+      "loss": 0.099,
+      "step": 1210
+    },
+    {
+      "epoch": 2.6248648648648647,
+      "grad_norm": 0.8788983225822449,
+      "learning_rate": 1.2842533959371066e-06,
+      "loss": 0.116,
+      "step": 1215
+    },
+    {
+      "epoch": 2.6356756756756754,
+      "grad_norm": 0.6774969100952148,
+      "learning_rate": 1.2129073592520685e-06,
+      "loss": 0.1077,
+      "step": 1220
+    },
+    {
+      "epoch": 2.6464864864864865,
+      "grad_norm": 0.8134023547172546,
+      "learning_rate": 1.1435166420334436e-06,
+      "loss": 0.0987,
+      "step": 1225
+    },
+    {
+      "epoch": 2.6572972972972972,
+      "grad_norm": 0.9484908580780029,
+      "learning_rate": 1.0760910854438104e-06,
+      "loss": 0.104,
+      "step": 1230
+    },
+    {
+      "epoch": 2.668108108108108,
+      "grad_norm": 1.2464313507080078,
+      "learning_rate": 1.0106402519418173e-06,
+      "loss": 0.1009,
+      "step": 1235
+    },
+    {
+      "epoch": 2.678918918918919,
+      "grad_norm": 0.9004675149917603,
+      "learning_rate": 9.471734239260288e-07,
+      "loss": 0.1111,
+      "step": 1240
+    },
+    {
+      "epoch": 2.68972972972973,
+      "grad_norm": 0.7725552916526794,
+      "learning_rate": 8.856996024184477e-07,
+      "loss": 0.0994,
+      "step": 1245
+    },
+    {
+      "epoch": 2.7005405405405405,
+      "grad_norm": 1.1223211288452148,
+      "learning_rate": 8.262275057879926e-07,
+      "loss": 0.0939,
+      "step": 1250
+    },
+    {
+      "epoch": 2.711351351351351,
+      "grad_norm": 0.8283373117446899,
+      "learning_rate": 7.6876556851401e-07,
+      "loss": 0.108,
+      "step": 1255
+    },
+    {
+      "epoch": 2.722162162162162,
+      "grad_norm": 0.9991348385810852,
+      "learning_rate": 7.133219399901097e-07,
+      "loss": 0.1004,
+      "step": 1260
+    },
+    {
+      "epoch": 2.732972972972973,
+      "grad_norm": 0.8724759221076965,
+      "learning_rate": 6.599044833683632e-07,
+      "loss": 0.099,
+      "step": 1265
+    },
+    {
+      "epoch": 2.7437837837837837,
+      "grad_norm": 0.7052216529846191,
+      "learning_rate": 6.085207744441529e-07,
+      "loss": 0.1017,
+      "step": 1270
+    },
+    {
+      "epoch": 2.7545945945945944,
+      "grad_norm": 0.71845543384552,
+      "learning_rate": 5.591781005817542e-07,
+      "loss": 0.1031,
+      "step": 1275
+    },
+    {
+      "epoch": 2.7654054054054056,
+      "grad_norm": 0.7921403050422668,
+      "learning_rate": 5.11883459680812e-07,
+      "loss": 0.1163,
+      "step": 1280
+    },
+    {
+      "epoch": 2.7762162162162163,
+      "grad_norm": 1.055810570716858,
+      "learning_rate": 4.6664355918389244e-07,
+      "loss": 0.1208,
+      "step": 1285
+    },
+    {
+      "epoch": 2.787027027027027,
+      "grad_norm": 0.9086781740188599,
+      "learning_rate": 4.234648151252063e-07,
+      "loss": 0.0968,
+      "step": 1290
+    },
+    {
+      "epoch": 2.7978378378378377,
+      "grad_norm": 0.786642849445343,
+      "learning_rate": 3.823533512206845e-07,
+      "loss": 0.0949,
+      "step": 1295
+    },
+    {
+      "epoch": 2.8086486486486484,
+      "grad_norm": 0.7988888621330261,
+      "learning_rate": 3.4331499799948484e-07,
+      "loss": 0.0983,
+      "step": 1300
+    },
+    {
+      "epoch": 2.8194594594594595,
+      "grad_norm": 0.8628491759300232,
+      "learning_rate": 3.063552919770984e-07,
+      "loss": 0.1036,
+      "step": 1305
+    },
+    {
+      "epoch": 2.8302702702702702,
+      "grad_norm": 0.9562894701957703,
+      "learning_rate": 2.7147947487014434e-07,
+      "loss": 0.1116,
+      "step": 1310
+    },
+    {
+      "epoch": 2.841081081081081,
+      "grad_norm": 0.8303439617156982,
+      "learning_rate": 2.38692492852986e-07,
+      "loss": 0.0981,
+      "step": 1315
+    },
+    {
+      "epoch": 2.851891891891892,
+      "grad_norm": 0.7052004933357239,
+      "learning_rate": 2.0799899585623894e-07,
+      "loss": 0.1065,
+      "step": 1320
+    },
+    {
+      "epoch": 2.8627027027027028,
+      "grad_norm": 0.8546080589294434,
+      "learning_rate": 1.7940333690731004e-07,
+      "loss": 0.1014,
+      "step": 1325
+    },
+    {
+      "epoch": 2.8735135135135135,
+      "grad_norm": 0.9116050601005554,
+      "learning_rate": 1.5290957151304795e-07,
+      "loss": 0.0956,
+      "step": 1330
+    },
+    {
+      "epoch": 2.884324324324324,
+      "grad_norm": 0.8937197923660278,
+      "learning_rate": 1.2852145708457498e-07,
+      "loss": 0.1015,
+      "step": 1335
+    },
+    {
+      "epoch": 2.895135135135135,
+      "grad_norm": 0.7416398525238037,
+      "learning_rate": 1.0624245240439811e-07,
+      "loss": 0.101,
+      "step": 1340
+    },
+    {
+      "epoch": 2.905945945945946,
+      "grad_norm": 0.8376498222351074,
+      "learning_rate": 8.607571713588502e-08,
+      "loss": 0.1156,
+      "step": 1345
+    },
+    {
+      "epoch": 2.9167567567567567,
+      "grad_norm": 0.795005738735199,
+      "learning_rate": 6.802411137514897e-08,
+      "loss": 0.0931,
+      "step": 1350
+    },
+    {
+      "epoch": 2.9275675675675674,
+      "grad_norm": 0.8672885298728943,
+      "learning_rate": 5.20901952454167e-08,
+      "loss": 0.1028,
+      "step": 1355
+    },
+    {
+      "epoch": 2.9383783783783786,
+      "grad_norm": 0.8281853795051575,
+      "learning_rate": 3.827622853395551e-08,
+      "loss": 0.1112,
+      "step": 1360
+    },
+    {
+      "epoch": 2.9491891891891893,
+      "grad_norm": 0.7827093005180359,
+      "learning_rate": 2.6584170371578008e-08,
+      "loss": 0.1049,
+      "step": 1365
+    },
+    {
+      "epoch": 2.96,
+      "grad_norm": 0.9035359025001526,
+      "learning_rate": 1.701567895479761e-08,
+      "loss": 0.1101,
+      "step": 1370
+    },
+    {
+      "epoch": 2.9708108108108107,
+      "grad_norm": 0.763901948928833,
+      "learning_rate": 9.572111310653387e-09,
+      "loss": 0.114,
+      "step": 1375
+    },
+    {
+      "epoch": 2.9816216216216214,
+      "grad_norm": 0.7890019416809082,
+      "learning_rate": 4.254523104260666e-09,
+      "loss": 0.0989,
+      "step": 1380
+    },
+    {
+      "epoch": 2.9924324324324325,
+      "grad_norm": 0.8369008898735046,
+      "learning_rate": 1.0636684890874548e-09,
+      "loss": 0.1054,
+      "step": 1385
+    },
+    {
+      "epoch": 3.0,
+      "step": 1389,
+      "total_flos": 1.8970279091854377e+18,
+      "train_loss": 0.46635930327050884,
+      "train_runtime": 1484.7838,
+      "train_samples_per_second": 29.893,
+      "train_steps_per_second": 0.935
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 1389,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 20,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.8970279091854377e+18,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

hotpotqa_test_knowledge_50_base/0_128_e3_3e-5/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bca2c8fad61cb5da023e117eb207700e10c3f0189451255b0c236d77f226fd46
+size 8273

hotpotqa_test_knowledge_50_base/1_128_e3_3e-5/README.md

@@ -0,0 +1,63 @@
+---
+library_name: peft
+license: llama3.1
+base_model: meta-llama/Llama-3.1-8B
+tags:
+- alignment-handbook
+- generated_from_trainer
+datasets:
+- data/hotpotqa/test/processed/knowledge_50
+model-index:
+- name: 1_128_e3_3e-5
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# 1_128_e3_3e-5
+
+This model is a fine-tuned version of [meta-llama/Llama-3.1-8B](https://huggingface.co/meta-llama/Llama-3.1-8B) on the data/hotpotqa/test/processed/knowledge_50 dataset.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 3e-05
+- train_batch_size: 2
+- eval_batch_size: 1
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 4
+- gradient_accumulation_steps: 4
+- total_train_batch_size: 32
+- total_eval_batch_size: 4
+- optimizer: Use adamw_torch with betas=(0.9,0.95) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.05
+- num_epochs: 3.0
+
+### Training results
+
+
+
+### Framework versions
+
+- PEFT 0.15.2
+- Transformers 4.52.4
+- Pytorch 2.7.0+cu126
+- Datasets 3.6.0
+- Tokenizers 0.21.2

hotpotqa_test_knowledge_50_base/1_128_e3_3e-5/adapter_config.json

@@ -0,0 +1,39 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "meta-llama/Llama-3.1-8B",
+  "bias": "none",
+  "corda_config": null,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 256,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 128,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "v_proj",
+    "k_proj",
+    "o_proj",
+    "up_proj",
+    "down_proj",
+    "q_proj",
+    "gate_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_rslora": false
+}

hotpotqa_test_knowledge_50_base/1_128_e3_3e-5/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fed4d41ecf7958a8a30a5c6c928360a38dd3d50f62f800bcd8d0ec4147294b27
+size 671150064

hotpotqa_test_knowledge_50_base/1_128_e3_3e-5/all_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "total_flos": 1.508233453999489e+18,
+    "train_loss": 0.4364819198012456,
+    "train_runtime": 1183.9617,
+    "train_samples": 12174,
+    "train_samples_per_second": 30.847,
+    "train_steps_per_second": 0.965
+}

hotpotqa_test_knowledge_50_base/1_128_e3_3e-5/config.json

@@ -0,0 +1,35 @@
+{
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 128000,
+  "eos_token_id": 128001,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 131072,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": {
+    "factor": 8.0,
+    "high_freq_factor": 4.0,
+    "low_freq_factor": 1.0,
+    "original_max_position_embeddings": 8192,
+    "rope_type": "llama3"
+  },
+  "rope_theta": 500000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.52.4",
+  "use_cache": true,
+  "vocab_size": 128256
+}

hotpotqa_test_knowledge_50_base/1_128_e3_3e-5/special_tokens_map.json

@@ -0,0 +1,17 @@
+{
+  "bos_token": {
+    "content": "<|begin_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|end_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<|reserved_special_token_247|>"
+}

hotpotqa_test_knowledge_50_base/1_128_e3_3e-5/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:384a7e7c676f7be2e5d2e8449c508be9b00e5b18c5b3c39ebc626e96b3f4b988
+size 17210019

hotpotqa_test_knowledge_50_base/1_128_e3_3e-5/tokenizer_config.json

@@ -0,0 +1,2063 @@
+{
+  "added_tokens_decoder": {
+    "128000": {
+      "content": "<|begin_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128001": {
+      "content": "<|end_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128002": {
+      "content": "<|reserved_special_token_0|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128003": {
+      "content": "<|reserved_special_token_1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128004": {
+      "content": "<|finetune_right_pad_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128005": {
+      "content": "<|reserved_special_token_2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128006": {
+      "content": "<|start_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128007": {
+      "content": "<|end_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128008": {
+      "content": "<|eom_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128009": {
+      "content": "<|eot_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128010": {
+      "content": "<|python_tag|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128011": {
+      "content": "<|reserved_special_token_3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128012": {
+      "content": "<|reserved_special_token_4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128013": {
+      "content": "<|reserved_special_token_5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128014": {
+      "content": "<|reserved_special_token_6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128015": {
+      "content": "<|reserved_special_token_7|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128016": {
+      "content": "<|reserved_special_token_8|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128017": {
+      "content": "<|reserved_special_token_9|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128018": {
+      "content": "<|reserved_special_token_10|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128019": {
+      "content": "<|reserved_special_token_11|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128020": {
+      "content": "<|reserved_special_token_12|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128021": {
+      "content": "<|reserved_special_token_13|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128022": {
+      "content": "<|reserved_special_token_14|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128023": {
+      "content": "<|reserved_special_token_15|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128024": {
+      "content": "<|reserved_special_token_16|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128025": {
+      "content": "<|reserved_special_token_17|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128026": {
+      "content": "<|reserved_special_token_18|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128027": {
+      "content": "<|reserved_special_token_19|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128028": {
+      "content": "<|reserved_special_token_20|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128029": {
+      "content": "<|reserved_special_token_21|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128030": {
+      "content": "<|reserved_special_token_22|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128031": {
+      "content": "<|reserved_special_token_23|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128032": {
+      "content": "<|reserved_special_token_24|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128033": {
+      "content": "<|reserved_special_token_25|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128034": {
+      "content": "<|reserved_special_token_26|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128035": {
+      "content": "<|reserved_special_token_27|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128036": {
+      "content": "<|reserved_special_token_28|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128037": {
+      "content": "<|reserved_special_token_29|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128038": {
+      "content": "<|reserved_special_token_30|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128039": {
+      "content": "<|reserved_special_token_31|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128040": {
+      "content": "<|reserved_special_token_32|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128041": {
+      "content": "<|reserved_special_token_33|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128042": {
+      "content": "<|reserved_special_token_34|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128043": {
+      "content": "<|reserved_special_token_35|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128044": {
+      "content": "<|reserved_special_token_36|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128045": {
+      "content": "<|reserved_special_token_37|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128046": {
+      "content": "<|reserved_special_token_38|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128047": {
+      "content": "<|reserved_special_token_39|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128048": {
+      "content": "<|reserved_special_token_40|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128049": {
+      "content": "<|reserved_special_token_41|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128050": {
+      "content": "<|reserved_special_token_42|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128051": {
+      "content": "<|reserved_special_token_43|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128052": {
+      "content": "<|reserved_special_token_44|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128053": {
+      "content": "<|reserved_special_token_45|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128054": {
+      "content": "<|reserved_special_token_46|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128055": {
+      "content": "<|reserved_special_token_47|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128056": {
+      "content": "<|reserved_special_token_48|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128057": {
+      "content": "<|reserved_special_token_49|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128058": {
+      "content": "<|reserved_special_token_50|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128059": {
+      "content": "<|reserved_special_token_51|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128060": {
+      "content": "<|reserved_special_token_52|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128061": {
+      "content": "<|reserved_special_token_53|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128062": {
+      "content": "<|reserved_special_token_54|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128063": {
+      "content": "<|reserved_special_token_55|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128064": {
+      "content": "<|reserved_special_token_56|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128065": {
+      "content": "<|reserved_special_token_57|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128066": {
+      "content": "<|reserved_special_token_58|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128067": {
+      "content": "<|reserved_special_token_59|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128068": {
+      "content": "<|reserved_special_token_60|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128069": {
+      "content": "<|reserved_special_token_61|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128070": {
+      "content": "<|reserved_special_token_62|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128071": {
+      "content": "<|reserved_special_token_63|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128072": {
+      "content": "<|reserved_special_token_64|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128073": {
+      "content": "<|reserved_special_token_65|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128074": {
+      "content": "<|reserved_special_token_66|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128075": {
+      "content": "<|reserved_special_token_67|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128076": {
+      "content": "<|reserved_special_token_68|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128077": {
+      "content": "<|reserved_special_token_69|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128078": {
+      "content": "<|reserved_special_token_70|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128079": {
+      "content": "<|reserved_special_token_71|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128080": {
+      "content": "<|reserved_special_token_72|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128081": {
+      "content": "<|reserved_special_token_73|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128082": {
+      "content": "<|reserved_special_token_74|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128083": {
+      "content": "<|reserved_special_token_75|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128084": {
+      "content": "<|reserved_special_token_76|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128085": {
+      "content": "<|reserved_special_token_77|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128086": {
+      "content": "<|reserved_special_token_78|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128087": {
+      "content": "<|reserved_special_token_79|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128088": {
+      "content": "<|reserved_special_token_80|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128089": {
+      "content": "<|reserved_special_token_81|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128090": {
+      "content": "<|reserved_special_token_82|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128091": {
+      "content": "<|reserved_special_token_83|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128092": {
+      "content": "<|reserved_special_token_84|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128093": {
+      "content": "<|reserved_special_token_85|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128094": {
+      "content": "<|reserved_special_token_86|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128095": {
+      "content": "<|reserved_special_token_87|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128096": {
+      "content": "<|reserved_special_token_88|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128097": {
+      "content": "<|reserved_special_token_89|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128098": {
+      "content": "<|reserved_special_token_90|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128099": {
+      "content": "<|reserved_special_token_91|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128100": {
+      "content": "<|reserved_special_token_92|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128101": {
+      "content": "<|reserved_special_token_93|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128102": {
+      "content": "<|reserved_special_token_94|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128103": {
+      "content": "<|reserved_special_token_95|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128104": {
+      "content": "<|reserved_special_token_96|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128105": {
+      "content": "<|reserved_special_token_97|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128106": {
+      "content": "<|reserved_special_token_98|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128107": {
+      "content": "<|reserved_special_token_99|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128108": {
+      "content": "<|reserved_special_token_100|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128109": {
+      "content": "<|reserved_special_token_101|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128110": {
+      "content": "<|reserved_special_token_102|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128111": {
+      "content": "<|reserved_special_token_103|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128112": {
+      "content": "<|reserved_special_token_104|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128113": {
+      "content": "<|reserved_special_token_105|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128114": {
+      "content": "<|reserved_special_token_106|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128115": {
+      "content": "<|reserved_special_token_107|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128116": {
+      "content": "<|reserved_special_token_108|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128117": {
+      "content": "<|reserved_special_token_109|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128118": {
+      "content": "<|reserved_special_token_110|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128119": {
+      "content": "<|reserved_special_token_111|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128120": {
+      "content": "<|reserved_special_token_112|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128121": {
+      "content": "<|reserved_special_token_113|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128122": {
+      "content": "<|reserved_special_token_114|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128123": {
+      "content": "<|reserved_special_token_115|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128124": {
+      "content": "<|reserved_special_token_116|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128125": {
+      "content": "<|reserved_special_token_117|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128126": {
+      "content": "<|reserved_special_token_118|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128127": {
+      "content": "<|reserved_special_token_119|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128128": {
+      "content": "<|reserved_special_token_120|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128129": {
+      "content": "<|reserved_special_token_121|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128130": {
+      "content": "<|reserved_special_token_122|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128131": {
+      "content": "<|reserved_special_token_123|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128132": {
+      "content": "<|reserved_special_token_124|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128133": {
+      "content": "<|reserved_special_token_125|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128134": {
+      "content": "<|reserved_special_token_126|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128135": {
+      "content": "<|reserved_special_token_127|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128136": {
+      "content": "<|reserved_special_token_128|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128137": {
+      "content": "<|reserved_special_token_129|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128138": {
+      "content": "<|reserved_special_token_130|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128139": {
+      "content": "<|reserved_special_token_131|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128140": {
+      "content": "<|reserved_special_token_132|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128141": {
+      "content": "<|reserved_special_token_133|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128142": {
+      "content": "<|reserved_special_token_134|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128143": {
+      "content": "<|reserved_special_token_135|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128144": {
+      "content": "<|reserved_special_token_136|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128145": {
+      "content": "<|reserved_special_token_137|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128146": {
+      "content": "<|reserved_special_token_138|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128147": {
+      "content": "<|reserved_special_token_139|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128148": {
+      "content": "<|reserved_special_token_140|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128149": {
+      "content": "<|reserved_special_token_141|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128150": {
+      "content": "<|reserved_special_token_142|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128151": {
+      "content": "<|reserved_special_token_143|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128152": {
+      "content": "<|reserved_special_token_144|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128153": {
+      "content": "<|reserved_special_token_145|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128154": {
+      "content": "<|reserved_special_token_146|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128155": {
+      "content": "<|reserved_special_token_147|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128156": {
+      "content": "<|reserved_special_token_148|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128157": {
+      "content": "<|reserved_special_token_149|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128158": {
+      "content": "<|reserved_special_token_150|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128159": {
+      "content": "<|reserved_special_token_151|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128160": {
+      "content": "<|reserved_special_token_152|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128161": {
+      "content": "<|reserved_special_token_153|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128162": {
+      "content": "<|reserved_special_token_154|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128163": {
+      "content": "<|reserved_special_token_155|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128164": {
+      "content": "<|reserved_special_token_156|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128165": {
+      "content": "<|reserved_special_token_157|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128166": {
+      "content": "<|reserved_special_token_158|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128167": {
+      "content": "<|reserved_special_token_159|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128168": {
+      "content": "<|reserved_special_token_160|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128169": {
+      "content": "<|reserved_special_token_161|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128170": {
+      "content": "<|reserved_special_token_162|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128171": {
+      "content": "<|reserved_special_token_163|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128172": {
+      "content": "<|reserved_special_token_164|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128173": {
+      "content": "<|reserved_special_token_165|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128174": {
+      "content": "<|reserved_special_token_166|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128175": {
+      "content": "<|reserved_special_token_167|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128176": {
+      "content": "<|reserved_special_token_168|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128177": {
+      "content": "<|reserved_special_token_169|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128178": {
+      "content": "<|reserved_special_token_170|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128179": {
+      "content": "<|reserved_special_token_171|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128180": {
+      "content": "<|reserved_special_token_172|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128181": {
+      "content": "<|reserved_special_token_173|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128182": {
+      "content": "<|reserved_special_token_174|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128183": {
+      "content": "<|reserved_special_token_175|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128184": {
+      "content": "<|reserved_special_token_176|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128185": {
+      "content": "<|reserved_special_token_177|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128186": {
+      "content": "<|reserved_special_token_178|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128187": {
+      "content": "<|reserved_special_token_179|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128188": {
+      "content": "<|reserved_special_token_180|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128189": {
+      "content": "<|reserved_special_token_181|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128190": {
+      "content": "<|reserved_special_token_182|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128191": {
+      "content": "<|reserved_special_token_183|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128192": {
+      "content": "<|reserved_special_token_184|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128193": {
+      "content": "<|reserved_special_token_185|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128194": {
+      "content": "<|reserved_special_token_186|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128195": {
+      "content": "<|reserved_special_token_187|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128196": {
+      "content": "<|reserved_special_token_188|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128197": {
+      "content": "<|reserved_special_token_189|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128198": {
+      "content": "<|reserved_special_token_190|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128199": {
+      "content": "<|reserved_special_token_191|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128200": {
+      "content": "<|reserved_special_token_192|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128201": {
+      "content": "<|reserved_special_token_193|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128202": {
+      "content": "<|reserved_special_token_194|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128203": {
+      "content": "<|reserved_special_token_195|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128204": {
+      "content": "<|reserved_special_token_196|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128205": {
+      "content": "<|reserved_special_token_197|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128206": {
+      "content": "<|reserved_special_token_198|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128207": {
+      "content": "<|reserved_special_token_199|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128208": {
+      "content": "<|reserved_special_token_200|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128209": {
+      "content": "<|reserved_special_token_201|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128210": {
+      "content": "<|reserved_special_token_202|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128211": {
+      "content": "<|reserved_special_token_203|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128212": {
+      "content": "<|reserved_special_token_204|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128213": {
+      "content": "<|reserved_special_token_205|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128214": {
+      "content": "<|reserved_special_token_206|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128215": {
+      "content": "<|reserved_special_token_207|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128216": {
+      "content": "<|reserved_special_token_208|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128217": {
+      "content": "<|reserved_special_token_209|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128218": {
+      "content": "<|reserved_special_token_210|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128219": {
+      "content": "<|reserved_special_token_211|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128220": {
+      "content": "<|reserved_special_token_212|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128221": {
+      "content": "<|reserved_special_token_213|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128222": {
+      "content": "<|reserved_special_token_214|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128223": {
+      "content": "<|reserved_special_token_215|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128224": {
+      "content": "<|reserved_special_token_216|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128225": {
+      "content": "<|reserved_special_token_217|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128226": {
+      "content": "<|reserved_special_token_218|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128227": {
+      "content": "<|reserved_special_token_219|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128228": {
+      "content": "<|reserved_special_token_220|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128229": {
+      "content": "<|reserved_special_token_221|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128230": {
+      "content": "<|reserved_special_token_222|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128231": {
+      "content": "<|reserved_special_token_223|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128232": {
+      "content": "<|reserved_special_token_224|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128233": {
+      "content": "<|reserved_special_token_225|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128234": {
+      "content": "<|reserved_special_token_226|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128235": {
+      "content": "<|reserved_special_token_227|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128236": {
+      "content": "<|reserved_special_token_228|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128237": {
+      "content": "<|reserved_special_token_229|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128238": {
+      "content": "<|reserved_special_token_230|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128239": {
+      "content": "<|reserved_special_token_231|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128240": {
+      "content": "<|reserved_special_token_232|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128241": {
+      "content": "<|reserved_special_token_233|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128242": {
+      "content": "<|reserved_special_token_234|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128243": {
+      "content": "<|reserved_special_token_235|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128244": {
+      "content": "<|reserved_special_token_236|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128245": {
+      "content": "<|reserved_special_token_237|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128246": {
+      "content": "<|reserved_special_token_238|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128247": {
+      "content": "<|reserved_special_token_239|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128248": {
+      "content": "<|reserved_special_token_240|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128249": {
+      "content": "<|reserved_special_token_241|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128250": {
+      "content": "<|reserved_special_token_242|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128251": {
+      "content": "<|reserved_special_token_243|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128252": {
+      "content": "<|reserved_special_token_244|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128253": {
+      "content": "<|reserved_special_token_245|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128254": {
+      "content": "<|reserved_special_token_246|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128255": {
+      "content": "<|reserved_special_token_247|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|begin_of_text|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|end_of_text|>",
+  "extra_special_tokens": {},
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 131072,
+  "pad_token": "<|reserved_special_token_247|>",
+  "tokenizer_class": "PreTrainedTokenizer"
+}

hotpotqa_test_knowledge_50_base/1_128_e3_3e-5/train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 3.0,
+    "total_flos": 1.508233453999489e+18,
+    "train_loss": 0.4364819198012456,
+    "train_runtime": 1183.9617,
+    "train_samples": 12174,
+    "train_samples_per_second": 30.847,
+    "train_steps_per_second": 0.965
+}

hotpotqa_test_knowledge_50_base/1_128_e3_3e-5/trainer_state.json

@@ -0,0 +1,1639 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 500,
+  "global_step": 1143,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.013140604467805518,
+      "grad_norm": 0.6000319719314575,
+      "learning_rate": 2.068965517241379e-06,
+      "loss": 1.4972,
+      "step": 5
+    },
+    {
+      "epoch": 0.026281208935611037,
+      "grad_norm": 0.5496951937675476,
+      "learning_rate": 4.655172413793104e-06,
+      "loss": 1.5573,
+      "step": 10
+    },
+    {
+      "epoch": 0.03942181340341656,
+      "grad_norm": 0.5497358441352844,
+      "learning_rate": 7.241379310344828e-06,
+      "loss": 1.5677,
+      "step": 15
+    },
+    {
+      "epoch": 0.052562417871222074,
+      "grad_norm": 0.47274050116539,
+      "learning_rate": 9.827586206896551e-06,
+      "loss": 1.5092,
+      "step": 20
+    },
+    {
+      "epoch": 0.0657030223390276,
+      "grad_norm": 0.5257453918457031,
+      "learning_rate": 1.2413793103448277e-05,
+      "loss": 1.5321,
+      "step": 25
+    },
+    {
+      "epoch": 0.07884362680683311,
+      "grad_norm": 0.49069836735725403,
+      "learning_rate": 1.5e-05,
+      "loss": 1.4581,
+      "step": 30
+    },
+    {
+      "epoch": 0.09198423127463863,
+      "grad_norm": 0.6515851616859436,
+      "learning_rate": 1.7586206896551724e-05,
+      "loss": 1.4119,
+      "step": 35
+    },
+    {
+      "epoch": 0.10512483574244415,
+      "grad_norm": 0.4764103591442108,
+      "learning_rate": 2.017241379310345e-05,
+      "loss": 1.4419,
+      "step": 40
+    },
+    {
+      "epoch": 0.11826544021024968,
+      "grad_norm": 0.49834540486335754,
+      "learning_rate": 2.275862068965517e-05,
+      "loss": 1.461,
+      "step": 45
+    },
+    {
+      "epoch": 0.1314060446780552,
+      "grad_norm": 0.5335782170295715,
+      "learning_rate": 2.5344827586206897e-05,
+      "loss": 1.3978,
+      "step": 50
+    },
+    {
+      "epoch": 0.1445466491458607,
+      "grad_norm": 0.5069740414619446,
+      "learning_rate": 2.793103448275862e-05,
+      "loss": 1.3683,
+      "step": 55
+    },
+    {
+      "epoch": 0.15768725361366623,
+      "grad_norm": 0.5438121557235718,
+      "learning_rate": 2.999993712163665e-05,
+      "loss": 1.3693,
+      "step": 60
+    },
+    {
+      "epoch": 0.17082785808147175,
+      "grad_norm": 0.631446897983551,
+      "learning_rate": 2.9997736434270605e-05,
+      "loss": 1.3212,
+      "step": 65
+    },
+    {
+      "epoch": 0.18396846254927726,
+      "grad_norm": 0.5521685481071472,
+      "learning_rate": 2.9992392355875752e-05,
+      "loss": 1.3559,
+      "step": 70
+    },
+    {
+      "epoch": 0.19710906701708278,
+      "grad_norm": 0.6499108076095581,
+      "learning_rate": 2.9983906006522986e-05,
+      "loss": 1.2451,
+      "step": 75
+    },
+    {
+      "epoch": 0.2102496714848883,
+      "grad_norm": 0.6416671872138977,
+      "learning_rate": 2.9972279164875014e-05,
+      "loss": 1.2699,
+      "step": 80
+    },
+    {
+      "epoch": 0.2233902759526938,
+      "grad_norm": 0.6594803333282471,
+      "learning_rate": 2.9957514267813553e-05,
+      "loss": 1.2558,
+      "step": 85
+    },
+    {
+      "epoch": 0.23653088042049936,
+      "grad_norm": 0.6575009822845459,
+      "learning_rate": 2.993961440992859e-05,
+      "loss": 1.2514,
+      "step": 90
+    },
+    {
+      "epoch": 0.24967148488830487,
+      "grad_norm": 0.6749443411827087,
+      "learning_rate": 2.9918583342869767e-05,
+      "loss": 1.1903,
+      "step": 95
+    },
+    {
+      "epoch": 0.2628120893561104,
+      "grad_norm": 0.7291126251220703,
+      "learning_rate": 2.989442547456011e-05,
+      "loss": 1.1976,
+      "step": 100
+    },
+    {
+      "epoch": 0.2759526938239159,
+      "grad_norm": 0.8062726855278015,
+      "learning_rate": 2.986714586827211e-05,
+      "loss": 1.1337,
+      "step": 105
+    },
+    {
+      "epoch": 0.2890932982917214,
+      "grad_norm": 0.7285217046737671,
+      "learning_rate": 2.9836750241566558e-05,
+      "loss": 1.1208,
+      "step": 110
+    },
+    {
+      "epoch": 0.30223390275952694,
+      "grad_norm": 0.7911546230316162,
+      "learning_rate": 2.9803244965094165e-05,
+      "loss": 1.0985,
+      "step": 115
+    },
+    {
+      "epoch": 0.31537450722733246,
+      "grad_norm": 0.8114403486251831,
+      "learning_rate": 2.976663706126034e-05,
+      "loss": 1.082,
+      "step": 120
+    },
+    {
+      "epoch": 0.328515111695138,
+      "grad_norm": 0.771315336227417,
+      "learning_rate": 2.972693420275336e-05,
+      "loss": 1.1007,
+      "step": 125
+    },
+    {
+      "epoch": 0.3416557161629435,
+      "grad_norm": 0.7282198667526245,
+      "learning_rate": 2.9684144710936236e-05,
+      "loss": 1.1099,
+      "step": 130
+    },
+    {
+      "epoch": 0.354796320630749,
+      "grad_norm": 0.9147740006446838,
+      "learning_rate": 2.9638277554102636e-05,
+      "loss": 1.0721,
+      "step": 135
+    },
+    {
+      "epoch": 0.3679369250985545,
+      "grad_norm": 0.8092377781867981,
+      "learning_rate": 2.9589342345597218e-05,
+      "loss": 1.0411,
+      "step": 140
+    },
+    {
+      "epoch": 0.38107752956636004,
+      "grad_norm": 0.8536360263824463,
+      "learning_rate": 2.953734934180073e-05,
+      "loss": 0.9824,
+      "step": 145
+    },
+    {
+      "epoch": 0.39421813403416556,
+      "grad_norm": 0.9517012238502502,
+      "learning_rate": 2.9482309439980404e-05,
+      "loss": 0.9272,
+      "step": 150
+    },
+    {
+      "epoch": 0.4073587385019711,
+      "grad_norm": 0.829008162021637,
+      "learning_rate": 2.9424234176005957e-05,
+      "loss": 1.0328,
+      "step": 155
+    },
+    {
+      "epoch": 0.4204993429697766,
+      "grad_norm": 0.9455755352973938,
+      "learning_rate": 2.9363135721931798e-05,
+      "loss": 0.9775,
+      "step": 160
+    },
+    {
+      "epoch": 0.4336399474375821,
+      "grad_norm": 0.9267396926879883,
+      "learning_rate": 2.9299026883445846e-05,
+      "loss": 0.964,
+      "step": 165
+    },
+    {
+      "epoch": 0.4467805519053876,
+      "grad_norm": 0.9962680339813232,
+      "learning_rate": 2.9231921097185604e-05,
+      "loss": 0.9039,
+      "step": 170
+    },
+    {
+      "epoch": 0.45992115637319314,
+      "grad_norm": 0.9546060562133789,
+      "learning_rate": 2.916183242792194e-05,
+      "loss": 0.9568,
+      "step": 175
+    },
+    {
+      "epoch": 0.4730617608409987,
+      "grad_norm": 1.0466433763504028,
+      "learning_rate": 2.9088775565611248e-05,
+      "loss": 0.8752,
+      "step": 180
+    },
+    {
+      "epoch": 0.48620236530880423,
+      "grad_norm": 0.9733150005340576,
+      "learning_rate": 2.901276582231656e-05,
+      "loss": 0.8709,
+      "step": 185
+    },
+    {
+      "epoch": 0.49934296977660975,
+      "grad_norm": 1.132362723350525,
+      "learning_rate": 2.893381912899826e-05,
+      "loss": 0.8662,
+      "step": 190
+    },
+    {
+      "epoch": 0.5124835742444153,
+      "grad_norm": 1.0795292854309082,
+      "learning_rate": 2.8851952032175136e-05,
+      "loss": 0.8259,
+      "step": 195
+    },
+    {
+      "epoch": 0.5256241787122208,
+      "grad_norm": 1.2181341648101807,
+      "learning_rate": 2.8767181690456345e-05,
+      "loss": 0.8681,
+      "step": 200
+    },
+    {
+      "epoch": 0.5387647831800263,
+      "grad_norm": 1.0852538347244263,
+      "learning_rate": 2.867952587094512e-05,
+      "loss": 0.8088,
+      "step": 205
+    },
+    {
+      "epoch": 0.5519053876478318,
+      "grad_norm": 0.9409130215644836,
+      "learning_rate": 2.8589002945514987e-05,
+      "loss": 0.8163,
+      "step": 210
+    },
+    {
+      "epoch": 0.5650459921156373,
+      "grad_norm": 1.0253455638885498,
+      "learning_rate": 2.8495631886959126e-05,
+      "loss": 0.7571,
+      "step": 215
+    },
+    {
+      "epoch": 0.5781865965834428,
+      "grad_norm": 1.0371079444885254,
+      "learning_rate": 2.8399432265013887e-05,
+      "loss": 0.7087,
+      "step": 220
+    },
+    {
+      "epoch": 0.5913272010512484,
+      "grad_norm": 1.0996919870376587,
+      "learning_rate": 2.8300424242257125e-05,
+      "loss": 0.7642,
+      "step": 225
+    },
+    {
+      "epoch": 0.6044678055190539,
+      "grad_norm": 1.1715346574783325,
+      "learning_rate": 2.8198628569882328e-05,
+      "loss": 0.7629,
+      "step": 230
+    },
+    {
+      "epoch": 0.6176084099868594,
+      "grad_norm": 1.1301043033599854,
+      "learning_rate": 2.809406658334933e-05,
+      "loss": 0.7807,
+      "step": 235
+    },
+    {
+      "epoch": 0.6307490144546649,
+      "grad_norm": 1.1367380619049072,
+      "learning_rate": 2.7986760197912594e-05,
+      "loss": 0.7807,
+      "step": 240
+    },
+    {
+      "epoch": 0.6438896189224704,
+      "grad_norm": 1.2245783805847168,
+      "learning_rate": 2.7876731904027994e-05,
+      "loss": 0.7522,
+      "step": 245
+    },
+    {
+      "epoch": 0.657030223390276,
+      "grad_norm": 1.1590607166290283,
+      "learning_rate": 2.7764004762638977e-05,
+      "loss": 0.669,
+      "step": 250
+    },
+    {
+      "epoch": 0.6701708278580815,
+      "grad_norm": 1.0896775722503662,
+      "learning_rate": 2.7648602400343235e-05,
+      "loss": 0.7041,
+      "step": 255
+    },
+    {
+      "epoch": 0.683311432325887,
+      "grad_norm": 1.1343854665756226,
+      "learning_rate": 2.7530549004440757e-05,
+      "loss": 0.7048,
+      "step": 260
+    },
+    {
+      "epoch": 0.6964520367936925,
+      "grad_norm": 1.302240252494812,
+      "learning_rate": 2.7409869317864406e-05,
+      "loss": 0.6561,
+      "step": 265
+    },
+    {
+      "epoch": 0.709592641261498,
+      "grad_norm": 1.1845916509628296,
+      "learning_rate": 2.7286588633994e-05,
+      "loss": 0.6793,
+      "step": 270
+    },
+    {
+      "epoch": 0.7227332457293035,
+      "grad_norm": 1.3149752616882324,
+      "learning_rate": 2.7160732791355076e-05,
+      "loss": 0.634,
+      "step": 275
+    },
+    {
+      "epoch": 0.735873850197109,
+      "grad_norm": 1.091576099395752,
+      "learning_rate": 2.7032328168203327e-05,
+      "loss": 0.664,
+      "step": 280
+    },
+    {
+      "epoch": 0.7490144546649146,
+      "grad_norm": 1.1638708114624023,
+      "learning_rate": 2.6901401676996e-05,
+      "loss": 0.6405,
+      "step": 285
+    },
+    {
+      "epoch": 0.7621550591327201,
+      "grad_norm": 1.1364482641220093,
+      "learning_rate": 2.6767980758751264e-05,
+      "loss": 0.6211,
+      "step": 290
+    },
+    {
+      "epoch": 0.7752956636005256,
+      "grad_norm": 1.2141324281692505,
+      "learning_rate": 2.6632093377296796e-05,
+      "loss": 0.6661,
+      "step": 295
+    },
+    {
+      "epoch": 0.7884362680683311,
+      "grad_norm": 1.4334216117858887,
+      "learning_rate": 2.649376801340887e-05,
+      "loss": 0.5666,
+      "step": 300
+    },
+    {
+      "epoch": 0.8015768725361366,
+      "grad_norm": 1.2394732236862183,
+      "learning_rate": 2.6353033658842996e-05,
+      "loss": 0.5991,
+      "step": 305
+    },
+    {
+      "epoch": 0.8147174770039421,
+      "grad_norm": 1.3286678791046143,
+      "learning_rate": 2.6209919810257514e-05,
+      "loss": 0.5626,
+      "step": 310
+    },
+    {
+      "epoch": 0.8278580814717477,
+      "grad_norm": 1.191128134727478,
+      "learning_rate": 2.606445646303138e-05,
+      "loss": 0.587,
+      "step": 315
+    },
+    {
+      "epoch": 0.8409986859395532,
+      "grad_norm": 1.3281301259994507,
+      "learning_rate": 2.591667410497738e-05,
+      "loss": 0.5776,
+      "step": 320
+    },
+    {
+      "epoch": 0.8541392904073587,
+      "grad_norm": 1.1658596992492676,
+      "learning_rate": 2.5766603709952184e-05,
+      "loss": 0.5748,
+      "step": 325
+    },
+    {
+      "epoch": 0.8672798948751642,
+      "grad_norm": 1.0998756885528564,
+      "learning_rate": 2.561427673136446e-05,
+      "loss": 0.5435,
+      "step": 330
+    },
+    {
+      "epoch": 0.8804204993429697,
+      "grad_norm": 1.2171285152435303,
+      "learning_rate": 2.5459725095582577e-05,
+      "loss": 0.591,
+      "step": 335
+    },
+    {
+      "epoch": 0.8935611038107752,
+      "grad_norm": 1.1682820320129395,
+      "learning_rate": 2.5302981195243083e-05,
+      "loss": 0.5622,
+      "step": 340
+    },
+    {
+      "epoch": 0.9067017082785808,
+      "grad_norm": 1.2696325778961182,
+      "learning_rate": 2.5144077882461516e-05,
+      "loss": 0.5228,
+      "step": 345
+    },
+    {
+      "epoch": 0.9198423127463863,
+      "grad_norm": 1.2294542789459229,
+      "learning_rate": 2.4983048461946893e-05,
+      "loss": 0.5008,
+      "step": 350
+    },
+    {
+      "epoch": 0.9329829172141918,
+      "grad_norm": 1.1982229948043823,
+      "learning_rate": 2.4819926684021342e-05,
+      "loss": 0.5068,
+      "step": 355
+    },
+    {
+      "epoch": 0.9461235216819974,
+      "grad_norm": 1.2754900455474854,
+      "learning_rate": 2.4654746737546328e-05,
+      "loss": 0.4912,
+      "step": 360
+    },
+    {
+      "epoch": 0.9592641261498029,
+      "grad_norm": 1.1950724124908447,
+      "learning_rate": 2.4487543242756993e-05,
+      "loss": 0.5285,
+      "step": 365
+    },
+    {
+      "epoch": 0.9724047306176085,
+      "grad_norm": 1.1647367477416992,
+      "learning_rate": 2.4318351244006055e-05,
+      "loss": 0.5249,
+      "step": 370
+    },
+    {
+      "epoch": 0.985545335085414,
+      "grad_norm": 1.2416152954101562,
+      "learning_rate": 2.4147206202418812e-05,
+      "loss": 0.4966,
+      "step": 375
+    },
+    {
+      "epoch": 0.9986859395532195,
+      "grad_norm": 1.1909600496292114,
+      "learning_rate": 2.3974143988460838e-05,
+      "loss": 0.4392,
+      "step": 380
+    },
+    {
+      "epoch": 1.0105124835742445,
+      "grad_norm": 1.1483696699142456,
+      "learning_rate": 2.3799200874419827e-05,
+      "loss": 0.4171,
+      "step": 385
+    },
+    {
+      "epoch": 1.02365308804205,
+      "grad_norm": 1.298833966255188,
+      "learning_rate": 2.3622413526803273e-05,
+      "loss": 0.3912,
+      "step": 390
+    },
+    {
+      "epoch": 1.0367936925098555,
+      "grad_norm": 1.4780399799346924,
+      "learning_rate": 2.3443818998653464e-05,
+      "loss": 0.4052,
+      "step": 395
+    },
+    {
+      "epoch": 1.049934296977661,
+      "grad_norm": 1.4238592386245728,
+      "learning_rate": 2.3263454721781537e-05,
+      "loss": 0.4899,
+      "step": 400
+    },
+    {
+      "epoch": 1.0630749014454666,
+      "grad_norm": 1.241113543510437,
+      "learning_rate": 2.308135849892208e-05,
+      "loss": 0.4632,
+      "step": 405
+    },
+    {
+      "epoch": 1.076215505913272,
+      "grad_norm": 1.2949522733688354,
+      "learning_rate": 2.2897568495810022e-05,
+      "loss": 0.3383,
+      "step": 410
+    },
+    {
+      "epoch": 1.0893561103810776,
+      "grad_norm": 1.3433641195297241,
+      "learning_rate": 2.271212323318144e-05,
+      "loss": 0.4163,
+      "step": 415
+    },
+    {
+      "epoch": 1.1024967148488831,
+      "grad_norm": 1.3279451131820679,
+      "learning_rate": 2.2525061578699962e-05,
+      "loss": 0.3814,
+      "step": 420
+    },
+    {
+      "epoch": 1.1156373193166886,
+      "grad_norm": 1.310359239578247,
+      "learning_rate": 2.233642273881045e-05,
+      "loss": 0.3702,
+      "step": 425
+    },
+    {
+      "epoch": 1.1287779237844942,
+      "grad_norm": 1.6063209772109985,
+      "learning_rate": 2.2146246250521677e-05,
+      "loss": 0.3602,
+      "step": 430
+    },
+    {
+      "epoch": 1.1419185282522997,
+      "grad_norm": 1.2085447311401367,
+      "learning_rate": 2.1954571973119726e-05,
+      "loss": 0.3885,
+      "step": 435
+    },
+    {
+      "epoch": 1.1550591327201052,
+      "grad_norm": 1.2866445779800415,
+      "learning_rate": 2.1761440079813845e-05,
+      "loss": 0.3897,
+      "step": 440
+    },
+    {
+      "epoch": 1.1681997371879107,
+      "grad_norm": 1.2492274045944214,
+      "learning_rate": 2.1566891049316515e-05,
+      "loss": 0.372,
+      "step": 445
+    },
+    {
+      "epoch": 1.1813403416557162,
+      "grad_norm": 1.3012983798980713,
+      "learning_rate": 2.137096565735943e-05,
+      "loss": 0.3672,
+      "step": 450
+    },
+    {
+      "epoch": 1.1944809461235217,
+      "grad_norm": 1.2833282947540283,
+      "learning_rate": 2.1173704968147327e-05,
+      "loss": 0.3211,
+      "step": 455
+    },
+    {
+      "epoch": 1.2076215505913273,
+      "grad_norm": 1.3121607303619385,
+      "learning_rate": 2.0975150325751262e-05,
+      "loss": 0.3702,
+      "step": 460
+    },
+    {
+      "epoch": 1.2207621550591328,
+      "grad_norm": 1.1029564142227173,
+      "learning_rate": 2.0775343345443267e-05,
+      "loss": 0.3338,
+      "step": 465
+    },
+    {
+      "epoch": 1.2339027595269383,
+      "grad_norm": 1.213159441947937,
+      "learning_rate": 2.057432590497418e-05,
+      "loss": 0.3315,
+      "step": 470
+    },
+    {
+      "epoch": 1.2470433639947438,
+      "grad_norm": 1.377474308013916,
+      "learning_rate": 2.0372140135796407e-05,
+      "loss": 0.373,
+      "step": 475
+    },
+    {
+      "epoch": 1.2601839684625493,
+      "grad_norm": 1.1862900257110596,
+      "learning_rate": 2.01688284142336e-05,
+      "loss": 0.3438,
+      "step": 480
+    },
+    {
+      "epoch": 1.2733245729303548,
+      "grad_norm": 1.2109731435775757,
+      "learning_rate": 1.9964433352598913e-05,
+      "loss": 0.323,
+      "step": 485
+    },
+    {
+      "epoch": 1.2864651773981604,
+      "grad_norm": 1.1861048936843872,
+      "learning_rate": 1.975899779026386e-05,
+      "loss": 0.3499,
+      "step": 490
+    },
+    {
+      "epoch": 1.2996057818659659,
+      "grad_norm": 1.3072450160980225,
+      "learning_rate": 1.955256478467959e-05,
+      "loss": 0.308,
+      "step": 495
+    },
+    {
+      "epoch": 1.3127463863337714,
+      "grad_norm": 1.4458948373794556,
+      "learning_rate": 1.9345177602352386e-05,
+      "loss": 0.3279,
+      "step": 500
+    },
+    {
+      "epoch": 1.325886990801577,
+      "grad_norm": 1.1371740102767944,
+      "learning_rate": 1.9136879709775424e-05,
+      "loss": 0.3243,
+      "step": 505
+    },
+    {
+      "epoch": 1.3390275952693824,
+      "grad_norm": 1.408117651939392,
+      "learning_rate": 1.8927714764318588e-05,
+      "loss": 0.3347,
+      "step": 510
+    },
+    {
+      "epoch": 1.352168199737188,
+      "grad_norm": 1.39098060131073,
+      "learning_rate": 1.871772660507826e-05,
+      "loss": 0.3161,
+      "step": 515
+    },
+    {
+      "epoch": 1.3653088042049935,
+      "grad_norm": 1.3169364929199219,
+      "learning_rate": 1.8506959243689043e-05,
+      "loss": 0.2942,
+      "step": 520
+    },
+    {
+      "epoch": 1.378449408672799,
+      "grad_norm": 1.1352839469909668,
+      "learning_rate": 1.829545685509934e-05,
+      "loss": 0.3084,
+      "step": 525
+    },
+    {
+      "epoch": 1.3915900131406045,
+      "grad_norm": 1.2993459701538086,
+      "learning_rate": 1.8083263768312645e-05,
+      "loss": 0.3009,
+      "step": 530
+    },
+    {
+      "epoch": 1.40473061760841,
+      "grad_norm": 1.206346869468689,
+      "learning_rate": 1.7870424457096593e-05,
+      "loss": 0.2734,
+      "step": 535
+    },
+    {
+      "epoch": 1.4178712220762155,
+      "grad_norm": 1.3716164827346802,
+      "learning_rate": 1.765698353066169e-05,
+      "loss": 0.2508,
+      "step": 540
+    },
+    {
+      "epoch": 1.431011826544021,
+      "grad_norm": 1.2523376941680908,
+      "learning_rate": 1.7442985724311566e-05,
+      "loss": 0.2903,
+      "step": 545
+    },
+    {
+      "epoch": 1.4441524310118266,
+      "grad_norm": 1.4354424476623535,
+      "learning_rate": 1.7228475890066908e-05,
+      "loss": 0.3164,
+      "step": 550
+    },
+    {
+      "epoch": 1.457293035479632,
+      "grad_norm": 1.2621711492538452,
+      "learning_rate": 1.7013498987264832e-05,
+      "loss": 0.2945,
+      "step": 555
+    },
+    {
+      "epoch": 1.4704336399474376,
+      "grad_norm": 1.4864765405654907,
+      "learning_rate": 1.6798100073135865e-05,
+      "loss": 0.2666,
+      "step": 560
+    },
+    {
+      "epoch": 1.483574244415243,
+      "grad_norm": 1.1217767000198364,
+      "learning_rate": 1.6582324293360298e-05,
+      "loss": 0.2586,
+      "step": 565
+    },
+    {
+      "epoch": 1.4967148488830486,
+      "grad_norm": 1.280014157295227,
+      "learning_rate": 1.6366216872606098e-05,
+      "loss": 0.2636,
+      "step": 570
+    },
+    {
+      "epoch": 1.5098554533508541,
+      "grad_norm": 1.2222352027893066,
+      "learning_rate": 1.6149823105050187e-05,
+      "loss": 0.2545,
+      "step": 575
+    },
+    {
+      "epoch": 1.5229960578186597,
+      "grad_norm": 1.3205738067626953,
+      "learning_rate": 1.5933188344885232e-05,
+      "loss": 0.2863,
+      "step": 580
+    },
+    {
+      "epoch": 1.5361366622864652,
+      "grad_norm": 1.3537684679031372,
+      "learning_rate": 1.5716357996813773e-05,
+      "loss": 0.2505,
+      "step": 585
+    },
+    {
+      "epoch": 1.5492772667542707,
+      "grad_norm": 1.2179908752441406,
+      "learning_rate": 1.5499377506531818e-05,
+      "loss": 0.2522,
+      "step": 590
+    },
+    {
+      "epoch": 1.5624178712220762,
+      "grad_norm": 1.3018555641174316,
+      "learning_rate": 1.5282292351203847e-05,
+      "loss": 0.2543,
+      "step": 595
+    },
+    {
+      "epoch": 1.5755584756898817,
+      "grad_norm": 1.1383919715881348,
+      "learning_rate": 1.5065148029931195e-05,
+      "loss": 0.2768,
+      "step": 600
+    },
+    {
+      "epoch": 1.5886990801576872,
+      "grad_norm": 1.245634913444519,
+      "learning_rate": 1.484799005421584e-05,
+      "loss": 0.2381,
+      "step": 605
+    },
+    {
+      "epoch": 1.6018396846254928,
+      "grad_norm": 1.0681072473526,
+      "learning_rate": 1.4630863938421603e-05,
+      "loss": 0.2664,
+      "step": 610
+    },
+    {
+      "epoch": 1.6149802890932983,
+      "grad_norm": 1.3179540634155273,
+      "learning_rate": 1.4413815190234777e-05,
+      "loss": 0.2588,
+      "step": 615
+    },
+    {
+      "epoch": 1.6281208935611038,
+      "grad_norm": 1.1153942346572876,
+      "learning_rate": 1.419688930112607e-05,
+      "loss": 0.2759,
+      "step": 620
+    },
+    {
+      "epoch": 1.6412614980289093,
+      "grad_norm": 1.6008579730987549,
+      "learning_rate": 1.3980131736816048e-05,
+      "loss": 0.2377,
+      "step": 625
+    },
+    {
+      "epoch": 1.6544021024967148,
+      "grad_norm": 1.1769739389419556,
+      "learning_rate": 1.3763587927745898e-05,
+      "loss": 0.2258,
+      "step": 630
+    },
+    {
+      "epoch": 1.6675427069645203,
+      "grad_norm": 1.1792428493499756,
+      "learning_rate": 1.3547303259555625e-05,
+      "loss": 0.2356,
+      "step": 635
+    },
+    {
+      "epoch": 1.6806833114323259,
+      "grad_norm": 1.315924048423767,
+      "learning_rate": 1.3331323063571647e-05,
+      "loss": 0.2526,
+      "step": 640
+    },
+    {
+      "epoch": 1.6938239159001314,
+      "grad_norm": 1.220938801765442,
+      "learning_rate": 1.3115692607305718e-05,
+      "loss": 0.2136,
+      "step": 645
+    },
+    {
+      "epoch": 1.7069645203679369,
+      "grad_norm": 1.235229253768921,
+      "learning_rate": 1.2900457084967302e-05,
+      "loss": 0.2392,
+      "step": 650
+    },
+    {
+      "epoch": 1.7201051248357424,
+      "grad_norm": 1.2829138040542603,
+      "learning_rate": 1.2685661607991238e-05,
+      "loss": 0.232,
+      "step": 655
+    },
+    {
+      "epoch": 1.733245729303548,
+      "grad_norm": 1.139499545097351,
+      "learning_rate": 1.2471351195582811e-05,
+      "loss": 0.1879,
+      "step": 660
+    },
+    {
+      "epoch": 1.7463863337713534,
+      "grad_norm": 1.2732905149459839,
+      "learning_rate": 1.2257570765282127e-05,
+      "loss": 0.2086,
+      "step": 665
+    },
+    {
+      "epoch": 1.759526938239159,
+      "grad_norm": 1.077530860900879,
+      "learning_rate": 1.2044365123549791e-05,
+      "loss": 0.2101,
+      "step": 670
+    },
+    {
+      "epoch": 1.7726675427069645,
+      "grad_norm": 1.1747920513153076,
+      "learning_rate": 1.183177895637589e-05,
+      "loss": 0.2287,
+      "step": 675
+    },
+    {
+      "epoch": 1.78580814717477,
+      "grad_norm": 1.350934624671936,
+      "learning_rate": 1.1619856819914186e-05,
+      "loss": 0.2231,
+      "step": 680
+    },
+    {
+      "epoch": 1.7989487516425755,
+      "grad_norm": 1.1574058532714844,
+      "learning_rate": 1.1408643131143566e-05,
+      "loss": 0.1953,
+      "step": 685
+    },
+    {
+      "epoch": 1.812089356110381,
+      "grad_norm": 1.9376565217971802,
+      "learning_rate": 1.1198182158558638e-05,
+      "loss": 0.1884,
+      "step": 690
+    },
+    {
+      "epoch": 1.8252299605781865,
+      "grad_norm": 1.1832637786865234,
+      "learning_rate": 1.098851801289144e-05,
+      "loss": 0.2011,
+      "step": 695
+    },
+    {
+      "epoch": 1.838370565045992,
+      "grad_norm": 1.1152653694152832,
+      "learning_rate": 1.0779694637866257e-05,
+      "loss": 0.1729,
+      "step": 700
+    },
+    {
+      "epoch": 1.8515111695137976,
+      "grad_norm": 1.2615498304367065,
+      "learning_rate": 1.0571755800989367e-05,
+      "loss": 0.1862,
+      "step": 705
+    },
+    {
+      "epoch": 1.864651773981603,
+      "grad_norm": 1.2463351488113403,
+      "learning_rate": 1.036474508437579e-05,
+      "loss": 0.1971,
+      "step": 710
+    },
+    {
+      "epoch": 1.8777923784494086,
+      "grad_norm": 1.2754261493682861,
+      "learning_rate": 1.0158705875614877e-05,
+      "loss": 0.1672,
+      "step": 715
+    },
+    {
+      "epoch": 1.8909329829172141,
+      "grad_norm": 1.2041103839874268,
+      "learning_rate": 9.953681358676622e-06,
+      "loss": 0.1718,
+      "step": 720
+    },
+    {
+      "epoch": 1.9040735873850196,
+      "grad_norm": 1.223314642906189,
+      "learning_rate": 9.749714504860753e-06,
+      "loss": 0.173,
+      "step": 725
+    },
+    {
+      "epoch": 1.9172141918528252,
+      "grad_norm": 1.2831774950027466,
+      "learning_rate": 9.54684806379026e-06,
+      "loss": 0.1908,
+      "step": 730
+    },
+    {
+      "epoch": 1.9303547963206307,
+      "grad_norm": 1.1934982538223267,
+      "learning_rate": 9.345124554451506e-06,
+      "loss": 0.1715,
+      "step": 735
+    },
+    {
+      "epoch": 1.9434954007884362,
+      "grad_norm": 1.3165861368179321,
+      "learning_rate": 9.144586256282619e-06,
+      "loss": 0.1691,
+      "step": 740
+    },
+    {
+      "epoch": 1.9566360052562417,
+      "grad_norm": 1.2910155057907104,
+      "learning_rate": 8.945275200312085e-06,
+      "loss": 0.1665,
+      "step": 745
+    },
+    {
+      "epoch": 1.9697766097240472,
+      "grad_norm": 1.0195131301879883,
+      "learning_rate": 8.747233160349434e-06,
+      "loss": 0.2066,
+      "step": 750
+    },
+    {
+      "epoch": 1.9829172141918527,
+      "grad_norm": 1.1912403106689453,
+      "learning_rate": 8.550501644229846e-06,
+      "loss": 0.1763,
+      "step": 755
+    },
+    {
+      "epoch": 1.9960578186596583,
+      "grad_norm": 1.1489989757537842,
+      "learning_rate": 8.355121885114439e-06,
+      "loss": 0.1569,
+      "step": 760
+    },
+    {
+      "epoch": 2.0078843626806835,
+      "grad_norm": 0.9988665580749512,
+      "learning_rate": 8.161134832848186e-06,
+      "loss": 0.1521,
+      "step": 765
+    },
+    {
+      "epoch": 2.021024967148489,
+      "grad_norm": 1.0327770709991455,
+      "learning_rate": 7.968581145377205e-06,
+      "loss": 0.1561,
+      "step": 770
+    },
+    {
+      "epoch": 2.0341655716162945,
+      "grad_norm": 0.9528421759605408,
+      "learning_rate": 7.777501180227199e-06,
+      "loss": 0.1392,
+      "step": 775
+    },
+    {
+      "epoch": 2.0473061760841,
+      "grad_norm": 1.0644992589950562,
+      "learning_rate": 7.587934986044916e-06,
+      "loss": 0.137,
+      "step": 780
+    },
+    {
+      "epoch": 2.0604467805519056,
+      "grad_norm": 1.3147528171539307,
+      "learning_rate": 7.3999222942042635e-06,
+      "loss": 0.141,
+      "step": 785
+    },
+    {
+      "epoch": 2.073587385019711,
+      "grad_norm": 1.1737008094787598,
+      "learning_rate": 7.213502510478993e-06,
+      "loss": 0.1148,
+      "step": 790
+    },
+    {
+      "epoch": 2.0867279894875166,
+      "grad_norm": 0.9637901186943054,
+      "learning_rate": 7.028714706783626e-06,
+      "loss": 0.1268,
+      "step": 795
+    },
+    {
+      "epoch": 2.099868593955322,
+      "grad_norm": 0.9228414297103882,
+      "learning_rate": 6.845597612984288e-06,
+      "loss": 0.1322,
+      "step": 800
+    },
+    {
+      "epoch": 2.1130091984231276,
+      "grad_norm": 0.8965508341789246,
+      "learning_rate": 6.664189608781295e-06,
+      "loss": 0.1472,
+      "step": 805
+    },
+    {
+      "epoch": 2.126149802890933,
+      "grad_norm": 1.1861528158187866,
+      "learning_rate": 6.484528715665131e-06,
+      "loss": 0.13,
+      "step": 810
+    },
+    {
+      "epoch": 2.1392904073587387,
+      "grad_norm": 0.9378185272216797,
+      "learning_rate": 6.306652588947454e-06,
+      "loss": 0.1244,
+      "step": 815
+    },
+    {
+      "epoch": 2.152431011826544,
+      "grad_norm": 1.1733484268188477,
+      "learning_rate": 6.130598509868895e-06,
+      "loss": 0.1194,
+      "step": 820
+    },
+    {
+      "epoch": 2.1655716162943497,
+      "grad_norm": 0.8082630038261414,
+      "learning_rate": 5.95640337778525e-06,
+      "loss": 0.1337,
+      "step": 825
+    },
+    {
+      "epoch": 2.178712220762155,
+      "grad_norm": 0.8850849866867065,
+      "learning_rate": 5.784103702433685e-06,
+      "loss": 0.1245,
+      "step": 830
+    },
+    {
+      "epoch": 2.1918528252299607,
+      "grad_norm": 1.1383304595947266,
+      "learning_rate": 5.613735596280661e-06,
+      "loss": 0.1305,
+      "step": 835
+    },
+    {
+      "epoch": 2.2049934296977662,
+      "grad_norm": 1.2028309106826782,
+      "learning_rate": 5.445334766953037e-06,
+      "loss": 0.1279,
+      "step": 840
+    },
+    {
+      "epoch": 2.2181340341655718,
+      "grad_norm": 0.9609809517860413,
+      "learning_rate": 5.278936509754112e-06,
+      "loss": 0.1321,
+      "step": 845
+    },
+    {
+      "epoch": 2.2312746386333773,
+      "grad_norm": 1.1674765348434448,
+      "learning_rate": 5.114575700266024e-06,
+      "loss": 0.117,
+      "step": 850
+    },
+    {
+      "epoch": 2.244415243101183,
+      "grad_norm": 0.9462134838104248,
+      "learning_rate": 4.95228678704014e-06,
+      "loss": 0.1091,
+      "step": 855
+    },
+    {
+      "epoch": 2.2575558475689883,
+      "grad_norm": 0.9706594944000244,
+      "learning_rate": 4.7921037843769614e-06,
+      "loss": 0.1166,
+      "step": 860
+    },
+    {
+      "epoch": 2.270696452036794,
+      "grad_norm": 1.1274670362472534,
+      "learning_rate": 4.6340602651970304e-06,
+      "loss": 0.1172,
+      "step": 865
+    },
+    {
+      "epoch": 2.2838370565045993,
+      "grad_norm": 1.4383842945098877,
+      "learning_rate": 4.478189354004334e-06,
+      "loss": 0.1325,
+      "step": 870
+    },
+    {
+      "epoch": 2.296977660972405,
+      "grad_norm": 0.9975386261940002,
+      "learning_rate": 4.324523719943716e-06,
+      "loss": 0.1352,
+      "step": 875
+    },
+    {
+      "epoch": 2.3101182654402104,
+      "grad_norm": 1.0504313707351685,
+      "learning_rate": 4.173095569953708e-06,
+      "loss": 0.103,
+      "step": 880
+    },
+    {
+      "epoch": 2.323258869908016,
+      "grad_norm": 1.1191004514694214,
+      "learning_rate": 4.023936642016266e-06,
+      "loss": 0.1276,
+      "step": 885
+    },
+    {
+      "epoch": 2.3363994743758214,
+      "grad_norm": 0.8486424088478088,
+      "learning_rate": 3.87707819850474e-06,
+      "loss": 0.118,
+      "step": 890
+    },
+    {
+      "epoch": 2.349540078843627,
+      "grad_norm": 1.0684998035430908,
+      "learning_rate": 3.7325510196315964e-06,
+      "loss": 0.1137,
+      "step": 895
+    },
+    {
+      "epoch": 2.3626806833114324,
+      "grad_norm": 0.7870803475379944,
+      "learning_rate": 3.5903853969971335e-06,
+      "loss": 0.089,
+      "step": 900
+    },
+    {
+      "epoch": 2.375821287779238,
+      "grad_norm": 1.016579270362854,
+      "learning_rate": 3.450611127240646e-06,
+      "loss": 0.1046,
+      "step": 905
+    },
+    {
+      "epoch": 2.3889618922470435,
+      "grad_norm": 1.0224157571792603,
+      "learning_rate": 3.313257505795317e-06,
+      "loss": 0.1013,
+      "step": 910
+    },
+    {
+      "epoch": 2.402102496714849,
+      "grad_norm": 0.7343006730079651,
+      "learning_rate": 3.1783533207481537e-06,
+      "loss": 0.0943,
+      "step": 915
+    },
+    {
+      "epoch": 2.4152431011826545,
+      "grad_norm": 1.0326985120773315,
+      "learning_rate": 3.045926846806277e-06,
+      "loss": 0.1095,
+      "step": 920
+    },
+    {
+      "epoch": 2.42838370565046,
+      "grad_norm": 0.957417368888855,
+      "learning_rate": 2.9160058393707656e-06,
+      "loss": 0.1114,
+      "step": 925
+    },
+    {
+      "epoch": 2.4415243101182655,
+      "grad_norm": 1.1126303672790527,
+      "learning_rate": 2.7886175287194142e-06,
+      "loss": 0.1006,
+      "step": 930
+    },
+    {
+      "epoch": 2.454664914586071,
+      "grad_norm": 0.755524218082428,
+      "learning_rate": 2.6637886142994725e-06,
+      "loss": 0.0962,
+      "step": 935
+    },
+    {
+      "epoch": 2.4678055190538766,
+      "grad_norm": 0.8578504323959351,
+      "learning_rate": 2.5415452591317023e-06,
+      "loss": 0.1128,
+      "step": 940
+    },
+    {
+      "epoch": 2.480946123521682,
+      "grad_norm": 0.9237407445907593,
+      "learning_rate": 2.4219130843268362e-06,
+      "loss": 0.1326,
+      "step": 945
+    },
+    {
+      "epoch": 2.4940867279894876,
+      "grad_norm": 0.7806246876716614,
+      "learning_rate": 2.304917163715636e-06,
+      "loss": 0.1042,
+      "step": 950
+    },
+    {
+      "epoch": 2.507227332457293,
+      "grad_norm": 0.7509581446647644,
+      "learning_rate": 2.1905820185936172e-06,
+      "loss": 0.0879,
+      "step": 955
+    },
+    {
+      "epoch": 2.5203679369250986,
+      "grad_norm": 1.1201022863388062,
+      "learning_rate": 2.0789316125816275e-06,
+      "loss": 0.1126,
+      "step": 960
+    },
+    {
+      "epoch": 2.533508541392904,
+      "grad_norm": 0.8480803966522217,
+      "learning_rate": 1.9699893466032733e-06,
+      "loss": 0.101,
+      "step": 965
+    },
+    {
+      "epoch": 2.5466491458607097,
+      "grad_norm": 0.7627381086349487,
+      "learning_rate": 1.8637780539803118e-06,
+      "loss": 0.1037,
+      "step": 970
+    },
+    {
+      "epoch": 2.559789750328515,
+      "grad_norm": 1.0410747528076172,
+      "learning_rate": 1.760319995646968e-06,
+      "loss": 0.0976,
+      "step": 975
+    },
+    {
+      "epoch": 2.5729303547963207,
+      "grad_norm": 0.8388499021530151,
+      "learning_rate": 1.6596368554842673e-06,
+      "loss": 0.109,
+      "step": 980
+    },
+    {
+      "epoch": 2.5860709592641262,
+      "grad_norm": 0.7287879586219788,
+      "learning_rate": 1.5617497357752724e-06,
+      "loss": 0.0952,
+      "step": 985
+    },
+    {
+      "epoch": 2.5992115637319317,
+      "grad_norm": 0.8424334526062012,
+      "learning_rate": 1.4666791527822377e-06,
+      "loss": 0.0934,
+      "step": 990
+    },
+    {
+      "epoch": 2.6123521681997373,
+      "grad_norm": 0.7682844996452332,
+      "learning_rate": 1.374445032446588e-06,
+      "loss": 0.0814,
+      "step": 995
+    },
+    {
+      "epoch": 2.6254927726675428,
+      "grad_norm": 0.7012391686439514,
+      "learning_rate": 1.285066706212612e-06,
+      "loss": 0.1003,
+      "step": 1000
+    },
+    {
+      "epoch": 2.6386333771353483,
+      "grad_norm": 0.7718296647071838,
+      "learning_rate": 1.1985629069757847e-06,
+      "loss": 0.1093,
+      "step": 1005
+    },
+    {
+      "epoch": 2.651773981603154,
+      "grad_norm": 0.8238117098808289,
+      "learning_rate": 1.1149517651564944e-06,
+      "loss": 0.1307,
+      "step": 1010
+    },
+    {
+      "epoch": 2.6649145860709593,
+      "grad_norm": 0.7402698993682861,
+      "learning_rate": 1.034250804900081e-06,
+      "loss": 0.1022,
+      "step": 1015
+    },
+    {
+      "epoch": 2.678055190538765,
+      "grad_norm": 0.9563916325569153,
+      "learning_rate": 9.56476940403942e-07,
+      "loss": 0.1193,
+      "step": 1020
+    },
+    {
+      "epoch": 2.6911957950065704,
+      "grad_norm": 0.8082496523857117,
+      "learning_rate": 8.816464723724504e-07,
+      "loss": 0.0984,
+      "step": 1025
+    },
+    {
+      "epoch": 2.704336399474376,
+      "grad_norm": 0.810088038444519,
+      "learning_rate": 8.097750846004909e-07,
+      "loss": 0.0873,
+      "step": 1030
+    },
+    {
+      "epoch": 2.7174770039421814,
+      "grad_norm": 0.8148086071014404,
+      "learning_rate": 7.40877840686257e-07,
+      "loss": 0.0998,
+      "step": 1035
+    },
+    {
+      "epoch": 2.730617608409987,
+      "grad_norm": 1.008510708808899,
+      "learning_rate": 6.749691808740777e-07,
+      "loss": 0.0973,
+      "step": 1040
+    },
+    {
+      "epoch": 2.7437582128777924,
+      "grad_norm": 0.5810213088989258,
+      "learning_rate": 6.120629190278554e-07,
+      "loss": 0.0787,
+      "step": 1045
+    },
+    {
+      "epoch": 2.756898817345598,
+      "grad_norm": 0.7217281460762024,
+      "learning_rate": 5.521722397358132e-07,
+      "loss": 0.0849,
+      "step": 1050
+    },
+    {
+      "epoch": 2.7700394218134035,
+      "grad_norm": 0.8091464042663574,
+      "learning_rate": 4.953096955471142e-07,
+      "loss": 0.0937,
+      "step": 1055
+    },
+    {
+      "epoch": 2.783180026281209,
+      "grad_norm": 0.7312224507331848,
+      "learning_rate": 4.414872043409757e-07,
+      "loss": 0.0951,
+      "step": 1060
+    },
+    {
+      "epoch": 2.7963206307490145,
+      "grad_norm": 0.834337592124939,
+      "learning_rate": 3.907160468287707e-07,
+      "loss": 0.1011,
+      "step": 1065
+    },
+    {
+      "epoch": 2.80946123521682,
+      "grad_norm": 0.7655376195907593,
+      "learning_rate": 3.4300686418970383e-07,
+      "loss": 0.0961,
+      "step": 1070
+    },
+    {
+      "epoch": 2.8226018396846255,
+      "grad_norm": 0.7533838152885437,
+      "learning_rate": 2.9836965584051303e-07,
+      "loss": 0.0909,
+      "step": 1075
+    },
+    {
+      "epoch": 2.835742444152431,
+      "grad_norm": 0.8285064101219177,
+      "learning_rate": 2.568137773396745e-07,
+      "loss": 0.0974,
+      "step": 1080
+    },
+    {
+      "epoch": 2.8488830486202366,
+      "grad_norm": 0.8482531309127808,
+      "learning_rate": 2.183479384265713e-07,
+      "loss": 0.0947,
+      "step": 1085
+    },
+    {
+      "epoch": 2.862023653088042,
+      "grad_norm": 0.7816241383552551,
+      "learning_rate": 1.8298020119600856e-07,
+      "loss": 0.0947,
+      "step": 1090
+    },
+    {
+      "epoch": 2.8751642575558476,
+      "grad_norm": 0.6632421612739563,
+      "learning_rate": 1.5071797840846336e-07,
+      "loss": 0.0974,
+      "step": 1095
+    },
+    {
+      "epoch": 2.888304862023653,
+      "grad_norm": 0.7103014588356018,
+      "learning_rate": 1.215680319364443e-07,
+      "loss": 0.0868,
+      "step": 1100
+    },
+    {
+      "epoch": 2.9014454664914586,
+      "grad_norm": 0.7530989646911621,
+      "learning_rate": 9.553647134726173e-08,
+      "loss": 0.093,
+      "step": 1105
+    },
+    {
+      "epoch": 2.914586070959264,
+      "grad_norm": 0.7976528406143188,
+      "learning_rate": 7.262875262251389e-08,
+      "loss": 0.0912,
+      "step": 1110
+    },
+    {
+      "epoch": 2.9277266754270697,
+      "grad_norm": 0.6478866338729858,
+      "learning_rate": 5.2849677014566445e-08,
+      "loss": 0.0896,
+      "step": 1115
+    },
+    {
+      "epoch": 2.940867279894875,
+      "grad_norm": 0.8027013540267944,
+      "learning_rate": 3.620339004025086e-08,
+      "loss": 0.0939,
+      "step": 1120
+    },
+    {
+      "epoch": 2.9540078843626807,
+      "grad_norm": 0.6769530773162842,
+      "learning_rate": 2.2693380612002767e-08,
+      "loss": 0.09,
+      "step": 1125
+    },
+    {
+      "epoch": 2.967148488830486,
+      "grad_norm": 0.7123112082481384,
+      "learning_rate": 1.2322480306615202e-08,
+      "loss": 0.0866,
+      "step": 1130
+    },
+    {
+      "epoch": 2.9802890932982917,
+      "grad_norm": 0.9540034532546997,
+      "learning_rate": 5.092862771765017e-09,
+      "loss": 0.1049,
+      "step": 1135
+    },
+    {
+      "epoch": 2.9934296977660972,
+      "grad_norm": 1.0238860845565796,
+      "learning_rate": 1.006043270438961e-09,
+      "loss": 0.1015,
+      "step": 1140
+    },
+    {
+      "epoch": 3.0,
+      "step": 1143,
+      "total_flos": 1.508233453999489e+18,
+      "train_loss": 0.4364819198012456,
+      "train_runtime": 1183.9617,
+      "train_samples_per_second": 30.847,
+      "train_steps_per_second": 0.965
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 1143,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 20,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.508233453999489e+18,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

hotpotqa_test_knowledge_50_base/1_128_e3_3e-5/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3fbd0eef003e2f5dd7807d65c62b330bfd1dbcc0887fd028a054614f6c7c2f5d
+size 8273

hotpotqa_test_knowledge_50_base/2_128_e3_3e-5/README.md

@@ -0,0 +1,63 @@
+---
+library_name: peft
+license: llama3.1
+base_model: meta-llama/Llama-3.1-8B
+tags:
+- alignment-handbook
+- generated_from_trainer
+datasets:
+- data/hotpotqa/test/processed/knowledge_50
+model-index:
+- name: 2_128_e3_3e-5
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# 2_128_e3_3e-5
+
+This model is a fine-tuned version of [meta-llama/Llama-3.1-8B](https://huggingface.co/meta-llama/Llama-3.1-8B) on the data/hotpotqa/test/processed/knowledge_50 dataset.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 3e-05
+- train_batch_size: 2
+- eval_batch_size: 1
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 4
+- gradient_accumulation_steps: 4
+- total_train_batch_size: 32
+- total_eval_batch_size: 4
+- optimizer: Use adamw_torch with betas=(0.9,0.95) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.05
+- num_epochs: 3.0
+
+### Training results
+
+
+
+### Framework versions
+
+- PEFT 0.15.2
+- Transformers 4.52.4
+- Pytorch 2.7.0+cu126
+- Datasets 3.6.0
+- Tokenizers 0.21.2

hotpotqa_test_knowledge_50_base/2_128_e3_3e-5/adapter_config.json

@@ -0,0 +1,39 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "meta-llama/Llama-3.1-8B",
+  "bias": "none",
+  "corda_config": null,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 256,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 128,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "o_proj",
+    "up_proj",
+    "k_proj",
+    "gate_proj",
+    "v_proj",
+    "down_proj",
+    "q_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_rslora": false
+}

hotpotqa_test_knowledge_50_base/2_128_e3_3e-5/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bee9c608d5aa05e65acb2d222a78ab062e4110749e2cc3159ea1b02a9b5b6e87
+size 671150064