Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

added_tokens.json

@@ -0,0 +1,289 @@
+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|/add_file|>": 151732,
+  "<|/arg_name|>": 151807,
+  "<|/arg_value|>": 151809,
+  "<|/assistant|>": 151679,
+  "<|/audio|>": 151897,
+  "<|/branch|>": 151696,
+  "<|/cite|>": 151876,
+  "<|/code|>": 151822,
+  "<|/commit_after|>": 151683,
+  "<|/commit_before|>": 151681,
+  "<|/commit_msg|>": 151685,
+  "<|/conclusion|>": 151859,
+  "<|/context|>": 151884,
+  "<|/correction|>": 151874,
+  "<|/delete_file|>": 151734,
+  "<|/diff|>": 151687,
+  "<|/doc|>": 151762,
+  "<|/edit_file|>": 151738,
+  "<|/edit_range|>": 151744,
+  "<|/emotion|>": 151899,
+  "<|/exec_error|>": 151729,
+  "<|/exec_result|>": 151727,
+  "<|/exec|>": 151725,
+  "<|/file:css|>": 151777,
+  "<|/file:csv|>": 151779,
+  "<|/file:html|>": 151775,
+  "<|/file:ini|>": 151783,
+  "<|/file:json|>": 151769,
+  "<|/file:log|>": 151785,
+  "<|/file:md|>": 151767,
+  "<|/file:toml|>": 151781,
+  "<|/file:txt|>": 151765,
+  "<|/file:xml|>": 151771,
+  "<|/file:yaml|>": 151773,
+  "<|/file_content|>": 151742,
+  "<|/filename|>": 151787,
+  "<|/filepath|>": 151789,
+  "<|/function_args|>": 151805,
+  "<|/function_def|>": 151813,
+  "<|/function_name|>": 151803,
+  "<|/gen_image|>": 151795,
+  "<|/gen_video|>": 151797,
+  "<|/gh_stars|>": 151694,
+  "<|/hypothesis|>": 151857,
+  "<|/image|>": 151791,
+  "<|/issue_body|>": 151702,
+  "<|/issue_title|>": 151700,
+  "<|/issue|>": 151698,
+  "<|/jupyter_code|>": 151716,
+  "<|/jupyter_error|>": 151722,
+  "<|/jupyter_output|>": 151718,
+  "<|/jupyter_text|>": 151720,
+  "<|/jupyter|>": 151714,
+  "<|/line|>": 151746,
+  "<|/listen|>": 151893,
+  "<|/note|>": 151851,
+  "<|/observation|>": 151849,
+  "<|/pr_body|>": 151710,
+  "<|/pr_title|>": 151708,
+  "<|/prompt|>": 151669,
+  "<|/prosody|>": 151913,
+  "<|/pr|>": 151706,
+  "<|/quote|>": 151882,
+  "<|/read_file|>": 151740,
+  "<|/reflection|>": 151855,
+  "<|/ref|>": 151880,
+  "<|/rename_file|>": 151736,
+  "<|/replace|>": 151750,
+  "<|/reponame|>": 151692,
+  "<|/response|>": 151673,
+  "<|/retrieved|>": 151886,
+  "<|/source|>": 151878,
+  "<|/speak|>": 151895,
+  "<|/step|>": 151853,
+  "<|/system|>": 151675,
+  "<|/text|>": 151671,
+  "<|/think|>": 151847,
+  "<|/tool_calls|>": 151818,
+  "<|/tool_call|>": 151799,
+  "<|/tool_error|>": 151815,
+  "<|/tool_id|>": 151820,
+  "<|/tool_result|>": 151801,
+  "<|/tools|>": 151811,
+  "<|/uncertain|>": 151865,
+  "<|/user|>": 151677,
+  "<|/verify|>": 151870,
+  "<|/video|>": 151793,
+  "<|add_file|>": 151731,
+  "<|arg_name|>": 151806,
+  "<|arg_value|>": 151808,
+  "<|assistant|>": 151678,
+  "<|audio|>": 151896,
+  "<|beyond_knowledge|>": 151890,
+  "<|bos|>": 151665,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|branch|>": 151695,
+  "<|cite|>": 151875,
+  "<|code|>": 151821,
+  "<|commit_after|>": 151682,
+  "<|commit_before|>": 151680,
+  "<|commit_msg|>": 151684,
+  "<|conclusion|>": 151858,
+  "<|confidence:high|>": 151860,
+  "<|confidence:low|>": 151862,
+  "<|confidence:medium|>": 151861,
+  "<|confidence:uncertain|>": 151863,
+  "<|context|>": 151883,
+  "<|correction|>": 151873,
+  "<|delete_file|>": 151733,
+  "<|diff_add|>": 151688,
+  "<|diff_ctx|>": 151690,
+  "<|diff_del|>": 151689,
+  "<|diff|>": 151686,
+  "<|doc|>": 151761,
+  "<|edit_file|>": 151737,
+  "<|edit_range|>": 151743,
+  "<|emotion:angry|>": 151903,
+  "<|emotion:calm|>": 151907,
+  "<|emotion:confident|>": 151911,
+  "<|emotion:curious|>": 151909,
+  "<|emotion:disgusted|>": 151906,
+  "<|emotion:excited|>": 151908,
+  "<|emotion:fearful|>": 151905,
+  "<|emotion:happy|>": 151901,
+  "<|emotion:neutral|>": 151900,
+  "<|emotion:sad|>": 151902,
+  "<|emotion:surprised|>": 151904,
+  "<|emotion:sympathetic|>": 151910,
+  "<|emotion|>": 151898,
+  "<|empty_output|>": 151723,
+  "<|endoftext|>": 151643,
+  "<|eod|>": 151763,
+  "<|eos|>": 151666,
+  "<|exec_error|>": 151728,
+  "<|exec_result|>": 151726,
+  "<|exec_timeout|>": 151730,
+  "<|exec|>": 151724,
+  "<|fact_check|>": 151871,
+  "<|file:css|>": 151776,
+  "<|file:csv|>": 151778,
+  "<|file:html|>": 151774,
+  "<|file:ini|>": 151782,
+  "<|file:json|>": 151768,
+  "<|file:log|>": 151784,
+  "<|file:md|>": 151766,
+  "<|file:toml|>": 151780,
+  "<|file:txt|>": 151764,
+  "<|file:xml|>": 151770,
+  "<|file:yaml|>": 151772,
+  "<|file_content|>": 151741,
+  "<|file_sep|>": 151664,
+  "<|filename|>": 151786,
+  "<|filepath|>": 151788,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|function_args|>": 151804,
+  "<|function_def|>": 151812,
+  "<|function_name|>": 151802,
+  "<|gen_image|>": 151794,
+  "<|gen_video|>": 151796,
+  "<|gh_stars|>": 151693,
+  "<|grounded|>": 151887,
+  "<|hypothesis|>": 151856,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|image|>": 151790,
+  "<|insert_after|>": 151748,
+  "<|insert_before|>": 151747,
+  "<|issue_body|>": 151701,
+  "<|issue_closed|>": 151703,
+  "<|issue_open|>": 151704,
+  "<|issue_title|>": 151699,
+  "<|issue|>": 151697,
+  "<|jupyter_code|>": 151715,
+  "<|jupyter_error|>": 151721,
+  "<|jupyter_output|>": 151717,
+  "<|jupyter_text|>": 151719,
+  "<|jupyter|>": 151713,
+  "<|knowledge_cutoff|>": 151889,
+  "<|lang:bash|>": 151838,
+  "<|lang:cpp|>": 151827,
+  "<|lang:csharp|>": 151829,
+  "<|lang:c|>": 151828,
+  "<|lang:go|>": 151830,
+  "<|lang:haskell|>": 151844,
+  "<|lang:javascript|>": 151824,
+  "<|lang:java|>": 151826,
+  "<|lang:kotlin|>": 151835,
+  "<|lang:lua|>": 151842,
+  "<|lang:matlab|>": 151841,
+  "<|lang:other|>": 151845,
+  "<|lang:perl|>": 151843,
+  "<|lang:php|>": 151833,
+  "<|lang:python|>": 151823,
+  "<|lang:ruby|>": 151832,
+  "<|lang:rust|>": 151831,
+  "<|lang:r|>": 151840,
+  "<|lang:scala|>": 151836,
+  "<|lang:shell|>": 151837,
+  "<|lang:sql|>": 151839,
+  "<|lang:swift|>": 151834,
+  "<|lang:typescript|>": 151825,
+  "<|line|>": 151745,
+  "<|listen|>": 151892,
+  "<|mask|>": 151929,
+  "<|may_outdated|>": 151868,
+  "<|need_verify|>": 151867,
+  "<|nl|>": 151928,
+  "<|note|>": 151850,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|observation|>": 151848,
+  "<|pad|>": 151667,
+  "<|pr_body|>": 151709,
+  "<|pr_closed|>": 151712,
+  "<|pr_merged|>": 151711,
+  "<|pr_title|>": 151707,
+  "<|prompt|>": 151668,
+  "<|prosody:emphasis|>": 151924,
+  "<|prosody:fast|>": 151914,
+  "<|prosody:high_pitch|>": 151921,
+  "<|prosody:loud|>": 151917,
+  "<|prosody:low_pitch|>": 151922,
+  "<|prosody:normal_pitch|>": 151923,
+  "<|prosody:normal_speed|>": 151916,
+  "<|prosody:normal_volume|>": 151920,
+  "<|prosody:pause_long|>": 151926,
+  "<|prosody:pause_short|>": 151925,
+  "<|prosody:slow|>": 151915,
+  "<|prosody:soft|>": 151918,
+  "<|prosody:whisper|>": 151919,
+  "<|prosody|>": 151912,
+  "<|pr|>": 151705,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|quote|>": 151881,
+  "<|read_file|>": 151739,
+  "<|reflection|>": 151854,
+  "<|ref|>": 151879,
+  "<|rename_file|>": 151735,
+  "<|replace|>": 151749,
+  "<|repo_name|>": 151663,
+  "<|reponame|>": 151691,
+  "<|response|>": 151672,
+  "<|retrieved|>": 151885,
+  "<|self_correct|>": 151872,
+  "<|sep|>": 151927,
+  "<|source|>": 151877,
+  "<|speak|>": 151894,
+  "<|step|>": 151852,
+  "<|system|>": 151674,
+  "<|text|>": 151670,
+  "<|think|>": 151846,
+  "<|tool_calls|>": 151817,
+  "<|tool_call|>": 151798,
+  "<|tool_error|>": 151814,
+  "<|tool_id|>": 151819,
+  "<|tool_result|>": 151800,
+  "<|tool_success|>": 151816,
+  "<|tools|>": 151810,
+  "<|uncertain|>": 151864,
+  "<|ungrounded|>": 151888,
+  "<|unknown|>": 151866,
+  "<|user|>": 151676,
+  "<|verify|>": 151869,
+  "<|video_pad|>": 151656,
+  "<|video|>": 151792,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652,
+  "<|within_knowledge|>": 151891,
+  "[d~[": 151753,
+  "[e~[": 151751,
+  "[m~[": 151759,
+  "[p~[": 151755,
+  "]~!b[": 151758,
+  "]~b]": 151757,
+  "]~d]": 151754,
+  "]~e]": 151752,
+  "]~m]": 151760,
+  "]~p]": 151756
+}

chat_template.jinja

@@ -0,0 +1,74 @@
+{%- set bos = '<|bos|>' -%}
+{%- set eos = '<|eos|>' -%}
+{%- set system_start = '<|system|>' -%}
+{%- set system_end = '<|/system|>' -%}
+{%- set user_start = '<|user|>' -%}
+{%- set user_end = '<|/user|>' -%}
+{%- set assistant_start = '<|assistant|>' -%}
+{%- set assistant_end = '<|/assistant|>' -%}
+{%- set image_start = '<|image|>' -%}
+{%- set image_end = '<|/image|>' -%}
+{%- set video_start = '<|video|>' -%}
+{%- set video_end = '<|/video|>' -%}
+{%- set audio_start = '<|audio|>' -%}
+{%- set audio_end = '<|/audio|>' -%}
+{%- set tool_call_start = '<|tool_call|>' -%}
+{%- set tool_call_end = '<|/tool_call|>' -%}
+{%- set tool_result_start = '<|tool_result|>' -%}
+{%- set tool_result_end = '<|/tool_result|>' -%}
+{%- set think_start = '<|think|>' -%}
+{%- set think_end = '<|/think|>' -%}
+{%- set code_start = '<|code|>' -%}
+{%- set code_end = '<|/code|>' -%}
+{%- set doc_start = '<|doc|>' -%}
+{%- set doc_end = '<|/doc|>' -%}
+{%- set eod = '<|eod|>' -%}
+
+{{- bos -}}
+{%- for message in messages -%}
+    {%- if message['role'] == 'system' -%}
+        {{- system_start + message['content'] + system_end -}}
+    {%- elif message['role'] == 'user' -%}
+        {{- user_start -}}
+        {%- if message.get('images') -%}
+            {%- for img in message['images'] -%}
+                {{- image_start + img + image_end -}}
+            {%- endfor -%}
+        {%- endif -%}
+        {%- if message.get('videos') -%}
+            {%- for vid in message['videos'] -%}
+                {{- video_start + vid + video_end -}}
+            {%- endfor -%}
+        {%- endif -%}
+        {%- if message.get('audio') -%}
+            {%- for aud in message['audio'] -%}
+                {{- audio_start + aud + audio_end -}}
+            {%- endfor -%}
+        {%- endif -%}
+        {%- if message.get('documents') -%}
+            {%- for doc in message['documents'] -%}
+                {{- doc_start + doc + doc_end -}}
+            {%- endfor -%}
+        {%- endif -%}
+        {{- message['content'] + user_end -}}
+    {%- elif message['role'] == 'assistant' -%}
+        {{- assistant_start -}}
+        {%- if message.get('thinking') -%}
+            {{- think_start + message['thinking'] + think_end -}}
+        {%- endif -%}
+        {%- if message.get('tool_calls') -%}
+            {%- for tool in message['tool_calls'] -%}
+                {{- tool_call_start + tool + tool_call_end -}}
+            {%- endfor -%}
+        {%- endif -%}
+        {{- message['content'] -}}
+        {%- if not loop.last or add_generation_prompt is not defined or not add_generation_prompt -%}
+            {{- assistant_end -}}
+        {%- endif -%}
+    {%- elif message['role'] == 'tool' -%}
+        {{- tool_result_start + message['content'] + tool_result_end -}}
+    {%- endif -%}
+{%- endfor -%}
+{%- if add_generation_prompt is defined and add_generation_prompt -%}
+    {{- assistant_start -}}
+{%- endif -%}

config.json

@@ -0,0 +1,50 @@
+{
+  "model_name": "Xoron-Dev-MultiMoE",
+  "hidden_size": 1024,
+  "num_layers": 12,
+  "num_heads": 16,
+  "intermediate_size": 2048,
+  "vocab_size": 151643,
+  "max_position_embeddings": 131072,
+  "use_sliding_window": true,
+  "sliding_window": 4096,
+  "use_moe": true,
+  "num_experts": 8,
+  "num_experts_per_tok": 2,
+  "moe_layer_freq": 2,
+  "router_aux_loss_coef": 0.1,
+  "vision_model_name": "openai/clip-vit-base-patch16",
+  "freeze_vision": false,
+  "num_vision_tokens": 64,
+  "max_video_frames": 32,
+  "enable_generation": true,
+  "generation_image_size": 256,
+  "generation_video_size": 256,
+  "generation_latent_channels": 4,
+  "generation_base_channels": 128,
+  "generation_inference_steps": 8,
+  "tokenizer_name": "Qwen/Qwen2.5-1.5B",
+  "use_lora": true,
+  "lora_r": 32,
+  "lora_alpha": 64,
+  "lora_dropout": 0.05,
+  "lora_target_modules": [
+    "q_proj",
+    "k_proj",
+    "v_proj",
+    "o_proj",
+    "gate_proj",
+    "up_proj",
+    "down_proj"
+  ],
+  "train_lora_only": false,
+  "use_cross_attention": true,
+  "cross_attention_layers": 4,
+  "cross_attention_heads": 8,
+  "cross_attention_dropout": 0.1,
+  "use_flash_attention": true,
+  "output_dir": "./xoron-model",
+  "has_audio_encoder": true,
+  "has_audio_decoder": true,
+  "lora_applied": true
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3d56b7fd05d000e5e3ada6f81f01c8327c38793e4ad5d48bb0f1bce97d256730
+size 3705372820

special_tokens.json

@@ -0,0 +1,272 @@
+{
+  "bos": "<|bos|>",
+  "eos": "<|eos|>",
+  "pad": "<|pad|>",
+  "prompt_start": "<|prompt|>",
+  "prompt_end": "<|/prompt|>",
+  "text_start": "<|text|>",
+  "text_end": "<|/text|>",
+  "response_start": "<|response|>",
+  "response_end": "<|/response|>",
+  "system_start": "<|system|>",
+  "system_end": "<|/system|>",
+  "user_start": "<|user|>",
+  "user_end": "<|/user|>",
+  "assistant_start": "<|assistant|>",
+  "assistant_end": "<|/assistant|>",
+  "fim_prefix": "<|fim_prefix|>",
+  "fim_middle": "<|fim_middle|>",
+  "fim_suffix": "<|fim_suffix|>",
+  "fim_pad": "<|fim_pad|>",
+  "commit_before": "<|commit_before|>",
+  "commit_before_end": "<|/commit_before|>",
+  "commit_after": "<|commit_after|>",
+  "commit_after_end": "<|/commit_after|>",
+  "commit_msg": "<|commit_msg|>",
+  "commit_msg_end": "<|/commit_msg|>",
+  "diff_start": "<|diff|>",
+  "diff_end": "<|/diff|>",
+  "diff_add": "<|diff_add|>",
+  "diff_del": "<|diff_del|>",
+  "diff_context": "<|diff_ctx|>",
+  "reponame": "<|reponame|>",
+  "reponame_end": "<|/reponame|>",
+  "gh_stars": "<|gh_stars|>",
+  "gh_stars_end": "<|/gh_stars|>",
+  "branch": "<|branch|>",
+  "branch_end": "<|/branch|>",
+  "issue_start": "<|issue|>",
+  "issue_end": "<|/issue|>",
+  "issue_title": "<|issue_title|>",
+  "issue_title_end": "<|/issue_title|>",
+  "issue_body": "<|issue_body|>",
+  "issue_body_end": "<|/issue_body|>",
+  "issue_closed": "<|issue_closed|>",
+  "issue_open": "<|issue_open|>",
+  "pr_start": "<|pr|>",
+  "pr_end": "<|/pr|>",
+  "pr_title": "<|pr_title|>",
+  "pr_title_end": "<|/pr_title|>",
+  "pr_body": "<|pr_body|>",
+  "pr_body_end": "<|/pr_body|>",
+  "pr_merged": "<|pr_merged|>",
+  "pr_closed": "<|pr_closed|>",
+  "jupyter_start": "<|jupyter|>",
+  "jupyter_end": "<|/jupyter|>",
+  "jupyter_code": "<|jupyter_code|>",
+  "jupyter_code_end": "<|/jupyter_code|>",
+  "jupyter_output": "<|jupyter_output|>",
+  "jupyter_output_end": "<|/jupyter_output|>",
+  "jupyter_text": "<|jupyter_text|>",
+  "jupyter_text_end": "<|/jupyter_text|>",
+  "jupyter_error": "<|jupyter_error|>",
+  "jupyter_error_end": "<|/jupyter_error|>",
+  "empty_output": "<|empty_output|>",
+  "exec_start": "<|exec|>",
+  "exec_end": "<|/exec|>",
+  "exec_result": "<|exec_result|>",
+  "exec_result_end": "<|/exec_result|>",
+  "exec_error": "<|exec_error|>",
+  "exec_error_end": "<|/exec_error|>",
+  "exec_timeout": "<|exec_timeout|>",
+  "add_file": "<|add_file|>",
+  "add_file_end": "<|/add_file|>",
+  "delete_file": "<|delete_file|>",
+  "delete_file_end": "<|/delete_file|>",
+  "rename_file": "<|rename_file|>",
+  "rename_file_end": "<|/rename_file|>",
+  "edit_file": "<|edit_file|>",
+  "edit_file_end": "<|/edit_file|>",
+  "read_file": "<|read_file|>",
+  "read_file_end": "<|/read_file|>",
+  "file_sep": "<|file_sep|>",
+  "file_content": "<|file_content|>",
+  "file_content_end": "<|/file_content|>",
+  "edit_range": "<|edit_range|>",
+  "edit_range_end": "<|/edit_range|>",
+  "line_num": "<|line|>",
+  "line_num_end": "<|/line|>",
+  "insert_before": "<|insert_before|>",
+  "insert_after": "<|insert_after|>",
+  "replace": "<|replace|>",
+  "replace_end": "<|/replace|>",
+  "encoder_start": "[e~[",
+  "encoder_end": "]~e]",
+  "decoder_start": "[d~[",
+  "decoder_end": "]~d]",
+  "projection_start": "[p~[",
+  "projection_end": "]~p]",
+  "state_begin": "]~b]",
+  "state_end": "]~!b[",
+  "modal_switch": "[m~[",
+  "modal_switch_end": "]~m]",
+  "doc_start": "<|doc|>",
+  "doc_end": "<|/doc|>",
+  "eod": "<|eod|>",
+  "file_txt": "<|file:txt|>",
+  "file_txt_end": "<|/file:txt|>",
+  "file_md": "<|file:md|>",
+  "file_md_end": "<|/file:md|>",
+  "file_json": "<|file:json|>",
+  "file_json_end": "<|/file:json|>",
+  "file_xml": "<|file:xml|>",
+  "file_xml_end": "<|/file:xml|>",
+  "file_yaml": "<|file:yaml|>",
+  "file_yaml_end": "<|/file:yaml|>",
+  "file_html": "<|file:html|>",
+  "file_html_end": "<|/file:html|>",
+  "file_css": "<|file:css|>",
+  "file_css_end": "<|/file:css|>",
+  "file_csv": "<|file:csv|>",
+  "file_csv_end": "<|/file:csv|>",
+  "file_toml": "<|file:toml|>",
+  "file_toml_end": "<|/file:toml|>",
+  "file_ini": "<|file:ini|>",
+  "file_ini_end": "<|/file:ini|>",
+  "file_log": "<|file:log|>",
+  "file_log_end": "<|/file:log|>",
+  "filename_start": "<|filename|>",
+  "filename_end": "<|/filename|>",
+  "filepath_start": "<|filepath|>",
+  "filepath_end": "<|/filepath|>",
+  "image_start": "<|image|>",
+  "image_end": "<|/image|>",
+  "video_start": "<|video|>",
+  "video_end": "<|/video|>",
+  "gen_image_start": "<|gen_image|>",
+  "gen_image_end": "<|/gen_image|>",
+  "gen_video_start": "<|gen_video|>",
+  "gen_video_end": "<|/gen_video|>",
+  "tool_call_start": "<|tool_call|>",
+  "tool_call_end": "<|/tool_call|>",
+  "tool_result_start": "<|tool_result|>",
+  "tool_result_end": "<|/tool_result|>",
+  "function_name_start": "<|function_name|>",
+  "function_name_end": "<|/function_name|>",
+  "function_args_start": "<|function_args|>",
+  "function_args_end": "<|/function_args|>",
+  "arg_name_start": "<|arg_name|>",
+  "arg_name_end": "<|/arg_name|>",
+  "arg_value_start": "<|arg_value|>",
+  "arg_value_end": "<|/arg_value|>",
+  "tools_start": "<|tools|>",
+  "tools_end": "<|/tools|>",
+  "function_def_start": "<|function_def|>",
+  "function_def_end": "<|/function_def|>",
+  "tool_error_start": "<|tool_error|>",
+  "tool_error_end": "<|/tool_error|>",
+  "tool_success": "<|tool_success|>",
+  "tool_calls_start": "<|tool_calls|>",
+  "tool_calls_end": "<|/tool_calls|>",
+  "tool_id_start": "<|tool_id|>",
+  "tool_id_end": "<|/tool_id|>",
+  "code_start": "<|code|>",
+  "code_end": "<|/code|>",
+  "lang_python": "<|lang:python|>",
+  "lang_javascript": "<|lang:javascript|>",
+  "lang_typescript": "<|lang:typescript|>",
+  "lang_java": "<|lang:java|>",
+  "lang_cpp": "<|lang:cpp|>",
+  "lang_c": "<|lang:c|>",
+  "lang_csharp": "<|lang:csharp|>",
+  "lang_go": "<|lang:go|>",
+  "lang_rust": "<|lang:rust|>",
+  "lang_ruby": "<|lang:ruby|>",
+  "lang_php": "<|lang:php|>",
+  "lang_swift": "<|lang:swift|>",
+  "lang_kotlin": "<|lang:kotlin|>",
+  "lang_scala": "<|lang:scala|>",
+  "lang_shell": "<|lang:shell|>",
+  "lang_bash": "<|lang:bash|>",
+  "lang_sql": "<|lang:sql|>",
+  "lang_r": "<|lang:r|>",
+  "lang_matlab": "<|lang:matlab|>",
+  "lang_lua": "<|lang:lua|>",
+  "lang_perl": "<|lang:perl|>",
+  "lang_haskell": "<|lang:haskell|>",
+  "lang_other": "<|lang:other|>",
+  "think_start": "<|think|>",
+  "think_end": "<|/think|>",
+  "observation_start": "<|observation|>",
+  "observation_end": "<|/observation|>",
+  "note_start": "<|note|>",
+  "note_end": "<|/note|>",
+  "step_start": "<|step|>",
+  "step_end": "<|/step|>",
+  "reflection_start": "<|reflection|>",
+  "reflection_end": "<|/reflection|>",
+  "hypothesis_start": "<|hypothesis|>",
+  "hypothesis_end": "<|/hypothesis|>",
+  "conclusion_start": "<|conclusion|>",
+  "conclusion_end": "<|/conclusion|>",
+  "confidence_high": "<|confidence:high|>",
+  "confidence_medium": "<|confidence:medium|>",
+  "confidence_low": "<|confidence:low|>",
+  "confidence_uncertain": "<|confidence:uncertain|>",
+  "uncertain_start": "<|uncertain|>",
+  "uncertain_end": "<|/uncertain|>",
+  "unknown": "<|unknown|>",
+  "need_verification": "<|need_verify|>",
+  "may_be_outdated": "<|may_outdated|>",
+  "verify_start": "<|verify|>",
+  "verify_end": "<|/verify|>",
+  "fact_check": "<|fact_check|>",
+  "self_correct": "<|self_correct|>",
+  "correction_start": "<|correction|>",
+  "correction_end": "<|/correction|>",
+  "cite_start": "<|cite|>",
+  "cite_end": "<|/cite|>",
+  "source_start": "<|source|>",
+  "source_end": "<|/source|>",
+  "reference_start": "<|ref|>",
+  "reference_end": "<|/ref|>",
+  "quote_start": "<|quote|>",
+  "quote_end": "<|/quote|>",
+  "context_start": "<|context|>",
+  "context_end": "<|/context|>",
+  "retrieved_start": "<|retrieved|>",
+  "retrieved_end": "<|/retrieved|>",
+  "grounded": "<|grounded|>",
+  "ungrounded": "<|ungrounded|>",
+  "knowledge_cutoff": "<|knowledge_cutoff|>",
+  "beyond_knowledge": "<|beyond_knowledge|>",
+  "within_knowledge": "<|within_knowledge|>",
+  "listen_start": "<|listen|>",
+  "listen_end": "<|/listen|>",
+  "speak_start": "<|speak|>",
+  "speak_end": "<|/speak|>",
+  "audio_start": "<|audio|>",
+  "audio_end": "<|/audio|>",
+  "emotion_start": "<|emotion|>",
+  "emotion_end": "<|/emotion|>",
+  "emotion_neutral": "<|emotion:neutral|>",
+  "emotion_happy": "<|emotion:happy|>",
+  "emotion_sad": "<|emotion:sad|>",
+  "emotion_angry": "<|emotion:angry|>",
+  "emotion_surprised": "<|emotion:surprised|>",
+  "emotion_fearful": "<|emotion:fearful|>",
+  "emotion_disgusted": "<|emotion:disgusted|>",
+  "emotion_calm": "<|emotion:calm|>",
+  "emotion_excited": "<|emotion:excited|>",
+  "emotion_curious": "<|emotion:curious|>",
+  "emotion_sympathetic": "<|emotion:sympathetic|>",
+  "emotion_confident": "<|emotion:confident|>",
+  "prosody_start": "<|prosody|>",
+  "prosody_end": "<|/prosody|>",
+  "prosody_fast": "<|prosody:fast|>",
+  "prosody_slow": "<|prosody:slow|>",
+  "prosody_normal_speed": "<|prosody:normal_speed|>",
+  "prosody_loud": "<|prosody:loud|>",
+  "prosody_soft": "<|prosody:soft|>",
+  "prosody_whisper": "<|prosody:whisper|>",
+  "prosody_normal_volume": "<|prosody:normal_volume|>",
+  "prosody_high_pitch": "<|prosody:high_pitch|>",
+  "prosody_low_pitch": "<|prosody:low_pitch|>",
+  "prosody_normal_pitch": "<|prosody:normal_pitch|>",
+  "prosody_emphasis": "<|prosody:emphasis|>",
+  "prosody_pause_short": "<|prosody:pause_short|>",
+  "prosody_pause_long": "<|prosody:pause_long|>",
+  "separator": "<|sep|>",
+  "newline": "<|nl|>",
+  "mask": "<|mask|>"
+}

special_tokens_map.json

@@ -0,0 +1,1897 @@
+{
+  "additional_special_tokens": [
+    {
+      "content": "<|bos|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|eos|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|prompt|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/prompt|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|response|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/response|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|system|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/system|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|user|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/user|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|assistant|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/assistant|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|commit_before|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/commit_before|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|commit_after|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/commit_after|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|commit_msg|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/commit_msg|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|diff|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/diff|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|diff_add|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|diff_del|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|diff_ctx|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|reponame|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/reponame|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|gh_stars|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/gh_stars|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|branch|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/branch|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|issue|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/issue|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|issue_title|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/issue_title|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|issue_body|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/issue_body|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|issue_closed|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|issue_open|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|pr|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/pr|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|pr_title|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/pr_title|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|pr_body|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/pr_body|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|pr_merged|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|pr_closed|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|jupyter|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/jupyter|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|jupyter_code|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/jupyter_code|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|jupyter_output|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/jupyter_output|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|jupyter_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/jupyter_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|jupyter_error|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/jupyter_error|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|empty_output|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|exec|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/exec|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|exec_result|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/exec_result|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|exec_error|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/exec_error|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|exec_timeout|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|add_file|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/add_file|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|delete_file|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/delete_file|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|rename_file|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/rename_file|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|edit_file|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/edit_file|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|read_file|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/read_file|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|file_content|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/file_content|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|edit_range|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/edit_range|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|line|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/line|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|insert_before|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|insert_after|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|replace|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/replace|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "[e~[",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "]~e]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "[d~[",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "]~d]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "[p~[",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "]~p]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "]~b]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "]~!b[",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "[m~[",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "]~m]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|doc|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/doc|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|eod|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|file:txt|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/file:txt|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|file:md|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/file:md|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|file:json|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/file:json|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|file:xml|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/file:xml|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|file:yaml|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/file:yaml|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|file:html|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/file:html|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|file:css|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/file:css|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|file:csv|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/file:csv|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|file:toml|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/file:toml|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|file:ini|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/file:ini|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|file:log|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/file:log|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|filename|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/filename|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|filepath|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/filepath|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|image|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/image|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|video|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/video|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|gen_image|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/gen_image|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|gen_video|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/gen_video|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|tool_call|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/tool_call|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|tool_result|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/tool_result|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|function_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/function_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|function_args|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/function_args|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|arg_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/arg_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|arg_value|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/arg_value|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|tools|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/tools|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|function_def|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/function_def|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|tool_error|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/tool_error|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|tool_success|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|tool_calls|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/tool_calls|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|tool_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/tool_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|code|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/code|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|lang:python|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|lang:javascript|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|lang:typescript|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|lang:java|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|lang:cpp|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|lang:c|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|lang:csharp|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|lang:go|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|lang:rust|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|lang:ruby|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|lang:php|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|lang:swift|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|lang:kotlin|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|lang:scala|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|lang:shell|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|lang:bash|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|lang:sql|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|lang:r|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|lang:matlab|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|lang:lua|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|lang:perl|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|lang:haskell|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|lang:other|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|think|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/think|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|observation|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/observation|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|note|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/note|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|step|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/step|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|reflection|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/reflection|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|hypothesis|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/hypothesis|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|conclusion|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/conclusion|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|confidence:high|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|confidence:medium|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|confidence:low|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|confidence:uncertain|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|uncertain|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/uncertain|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|unknown|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|need_verify|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|may_outdated|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|verify|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/verify|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|fact_check|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|self_correct|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|correction|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/correction|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|cite|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/cite|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|source|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/source|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|ref|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/ref|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|quote|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/quote|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|context|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/context|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|retrieved|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/retrieved|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|grounded|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|ungrounded|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|knowledge_cutoff|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|beyond_knowledge|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|within_knowledge|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|listen|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/listen|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|speak|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/speak|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|audio|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/audio|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|emotion|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/emotion|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|emotion:neutral|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|emotion:happy|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|emotion:sad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|emotion:angry|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|emotion:surprised|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|emotion:fearful|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|emotion:disgusted|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|emotion:calm|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|emotion:excited|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|emotion:curious|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|emotion:sympathetic|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|emotion:confident|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|prosody|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|/prosody|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|prosody:fast|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|prosody:slow|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|prosody:normal_speed|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|prosody:loud|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|prosody:soft|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|prosody:whisper|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|prosody:normal_volume|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|prosody:high_pitch|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|prosody:low_pitch|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|prosody:normal_pitch|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|prosody:emphasis|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|prosody:pause_short|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|prosody:pause_long|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|nl|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|mask|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    }
+  ],
+  "bos_token": "<|bos|>",
+  "eos_token": "<|eos|>",
+  "pad_token": "<|pad|>"
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:07134de730f10d8b2931f89cd086b374dbbba1a3ce6d2431b38d94584daad60a
+size 11472555

tokenizer_config.json

@@ -0,0 +1,2584 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151665": {
+      "content": "<|bos|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151666": {
+      "content": "<|eos|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151667": {
+      "content": "<|pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151668": {
+      "content": "<|prompt|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151669": {
+      "content": "<|/prompt|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151670": {
+      "content": "<|text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151671": {
+      "content": "<|/text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151672": {
+      "content": "<|response|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151673": {
+      "content": "<|/response|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151674": {
+      "content": "<|system|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151675": {
+      "content": "<|/system|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151676": {
+      "content": "<|user|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151677": {
+      "content": "<|/user|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151678": {
+      "content": "<|assistant|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151679": {
+      "content": "<|/assistant|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151680": {
+      "content": "<|commit_before|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151681": {
+      "content": "<|/commit_before|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151682": {
+      "content": "<|commit_after|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151683": {
+      "content": "<|/commit_after|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151684": {
+      "content": "<|commit_msg|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151685": {
+      "content": "<|/commit_msg|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151686": {
+      "content": "<|diff|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151687": {
+      "content": "<|/diff|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151688": {
+      "content": "<|diff_add|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151689": {
+      "content": "<|diff_del|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151690": {
+      "content": "<|diff_ctx|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151691": {
+      "content": "<|reponame|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151692": {
+      "content": "<|/reponame|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151693": {
+      "content": "<|gh_stars|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151694": {
+      "content": "<|/gh_stars|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151695": {
+      "content": "<|branch|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151696": {
+      "content": "<|/branch|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151697": {
+      "content": "<|issue|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151698": {
+      "content": "<|/issue|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151699": {
+      "content": "<|issue_title|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151700": {
+      "content": "<|/issue_title|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151701": {
+      "content": "<|issue_body|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151702": {
+      "content": "<|/issue_body|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151703": {
+      "content": "<|issue_closed|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151704": {
+      "content": "<|issue_open|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151705": {
+      "content": "<|pr|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151706": {
+      "content": "<|/pr|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151707": {
+      "content": "<|pr_title|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151708": {
+      "content": "<|/pr_title|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151709": {
+      "content": "<|pr_body|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151710": {
+      "content": "<|/pr_body|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151711": {
+      "content": "<|pr_merged|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151712": {
+      "content": "<|pr_closed|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151713": {
+      "content": "<|jupyter|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151714": {
+      "content": "<|/jupyter|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151715": {
+      "content": "<|jupyter_code|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151716": {
+      "content": "<|/jupyter_code|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151717": {
+      "content": "<|jupyter_output|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151718": {
+      "content": "<|/jupyter_output|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151719": {
+      "content": "<|jupyter_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151720": {
+      "content": "<|/jupyter_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151721": {
+      "content": "<|jupyter_error|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151722": {
+      "content": "<|/jupyter_error|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151723": {
+      "content": "<|empty_output|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151724": {
+      "content": "<|exec|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151725": {
+      "content": "<|/exec|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151726": {
+      "content": "<|exec_result|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151727": {
+      "content": "<|/exec_result|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151728": {
+      "content": "<|exec_error|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151729": {
+      "content": "<|/exec_error|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151730": {
+      "content": "<|exec_timeout|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151731": {
+      "content": "<|add_file|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151732": {
+      "content": "<|/add_file|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151733": {
+      "content": "<|delete_file|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151734": {
+      "content": "<|/delete_file|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151735": {
+      "content": "<|rename_file|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151736": {
+      "content": "<|/rename_file|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151737": {
+      "content": "<|edit_file|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151738": {
+      "content": "<|/edit_file|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151739": {
+      "content": "<|read_file|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151740": {
+      "content": "<|/read_file|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151741": {
+      "content": "<|file_content|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151742": {
+      "content": "<|/file_content|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151743": {
+      "content": "<|edit_range|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151744": {
+      "content": "<|/edit_range|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151745": {
+      "content": "<|line|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151746": {
+      "content": "<|/line|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151747": {
+      "content": "<|insert_before|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151748": {
+      "content": "<|insert_after|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151749": {
+      "content": "<|replace|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151750": {
+      "content": "<|/replace|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151751": {
+      "content": "[e~[",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151752": {
+      "content": "]~e]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151753": {
+      "content": "[d~[",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151754": {
+      "content": "]~d]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151755": {
+      "content": "[p~[",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151756": {
+      "content": "]~p]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151757": {
+      "content": "]~b]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151758": {
+      "content": "]~!b[",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151759": {
+      "content": "[m~[",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151760": {
+      "content": "]~m]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151761": {
+      "content": "<|doc|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151762": {
+      "content": "<|/doc|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151763": {
+      "content": "<|eod|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151764": {
+      "content": "<|file:txt|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151765": {
+      "content": "<|/file:txt|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151766": {
+      "content": "<|file:md|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151767": {
+      "content": "<|/file:md|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151768": {
+      "content": "<|file:json|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151769": {
+      "content": "<|/file:json|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151770": {
+      "content": "<|file:xml|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151771": {
+      "content": "<|/file:xml|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151772": {
+      "content": "<|file:yaml|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151773": {
+      "content": "<|/file:yaml|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151774": {
+      "content": "<|file:html|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151775": {
+      "content": "<|/file:html|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151776": {
+      "content": "<|file:css|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151777": {
+      "content": "<|/file:css|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151778": {
+      "content": "<|file:csv|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151779": {
+      "content": "<|/file:csv|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151780": {
+      "content": "<|file:toml|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151781": {
+      "content": "<|/file:toml|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151782": {
+      "content": "<|file:ini|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151783": {
+      "content": "<|/file:ini|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151784": {
+      "content": "<|file:log|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151785": {
+      "content": "<|/file:log|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151786": {
+      "content": "<|filename|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151787": {
+      "content": "<|/filename|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151788": {
+      "content": "<|filepath|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151789": {
+      "content": "<|/filepath|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151790": {
+      "content": "<|image|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151791": {
+      "content": "<|/image|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151792": {
+      "content": "<|video|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151793": {
+      "content": "<|/video|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151794": {
+      "content": "<|gen_image|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151795": {
+      "content": "<|/gen_image|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151796": {
+      "content": "<|gen_video|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151797": {
+      "content": "<|/gen_video|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151798": {
+      "content": "<|tool_call|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151799": {
+      "content": "<|/tool_call|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151800": {
+      "content": "<|tool_result|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151801": {
+      "content": "<|/tool_result|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151802": {
+      "content": "<|function_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151803": {
+      "content": "<|/function_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151804": {
+      "content": "<|function_args|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151805": {
+      "content": "<|/function_args|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151806": {
+      "content": "<|arg_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151807": {
+      "content": "<|/arg_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151808": {
+      "content": "<|arg_value|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151809": {
+      "content": "<|/arg_value|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151810": {
+      "content": "<|tools|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151811": {
+      "content": "<|/tools|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151812": {
+      "content": "<|function_def|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151813": {
+      "content": "<|/function_def|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151814": {
+      "content": "<|tool_error|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151815": {
+      "content": "<|/tool_error|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151816": {
+      "content": "<|tool_success|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151817": {
+      "content": "<|tool_calls|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151818": {
+      "content": "<|/tool_calls|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151819": {
+      "content": "<|tool_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151820": {
+      "content": "<|/tool_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151821": {
+      "content": "<|code|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151822": {
+      "content": "<|/code|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151823": {
+      "content": "<|lang:python|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151824": {
+      "content": "<|lang:javascript|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151825": {
+      "content": "<|lang:typescript|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151826": {
+      "content": "<|lang:java|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151827": {
+      "content": "<|lang:cpp|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151828": {
+      "content": "<|lang:c|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151829": {
+      "content": "<|lang:csharp|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151830": {
+      "content": "<|lang:go|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151831": {
+      "content": "<|lang:rust|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151832": {
+      "content": "<|lang:ruby|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151833": {
+      "content": "<|lang:php|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151834": {
+      "content": "<|lang:swift|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151835": {
+      "content": "<|lang:kotlin|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151836": {
+      "content": "<|lang:scala|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151837": {
+      "content": "<|lang:shell|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151838": {
+      "content": "<|lang:bash|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151839": {
+      "content": "<|lang:sql|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151840": {
+      "content": "<|lang:r|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151841": {
+      "content": "<|lang:matlab|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151842": {
+      "content": "<|lang:lua|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151843": {
+      "content": "<|lang:perl|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151844": {
+      "content": "<|lang:haskell|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151845": {
+      "content": "<|lang:other|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151846": {
+      "content": "<|think|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151847": {
+      "content": "<|/think|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151848": {
+      "content": "<|observation|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151849": {
+      "content": "<|/observation|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151850": {
+      "content": "<|note|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151851": {
+      "content": "<|/note|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151852": {
+      "content": "<|step|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151853": {
+      "content": "<|/step|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151854": {
+      "content": "<|reflection|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151855": {
+      "content": "<|/reflection|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151856": {
+      "content": "<|hypothesis|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151857": {
+      "content": "<|/hypothesis|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151858": {
+      "content": "<|conclusion|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151859": {
+      "content": "<|/conclusion|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151860": {
+      "content": "<|confidence:high|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151861": {
+      "content": "<|confidence:medium|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151862": {
+      "content": "<|confidence:low|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151863": {
+      "content": "<|confidence:uncertain|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151864": {
+      "content": "<|uncertain|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151865": {
+      "content": "<|/uncertain|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151866": {
+      "content": "<|unknown|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151867": {
+      "content": "<|need_verify|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151868": {
+      "content": "<|may_outdated|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151869": {
+      "content": "<|verify|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151870": {
+      "content": "<|/verify|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151871": {
+      "content": "<|fact_check|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151872": {
+      "content": "<|self_correct|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151873": {
+      "content": "<|correction|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151874": {
+      "content": "<|/correction|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151875": {
+      "content": "<|cite|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151876": {
+      "content": "<|/cite|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151877": {
+      "content": "<|source|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151878": {
+      "content": "<|/source|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151879": {
+      "content": "<|ref|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151880": {
+      "content": "<|/ref|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151881": {
+      "content": "<|quote|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151882": {
+      "content": "<|/quote|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151883": {
+      "content": "<|context|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151884": {
+      "content": "<|/context|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151885": {
+      "content": "<|retrieved|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151886": {
+      "content": "<|/retrieved|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151887": {
+      "content": "<|grounded|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151888": {
+      "content": "<|ungrounded|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151889": {
+      "content": "<|knowledge_cutoff|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151890": {
+      "content": "<|beyond_knowledge|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151891": {
+      "content": "<|within_knowledge|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151892": {
+      "content": "<|listen|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151893": {
+      "content": "<|/listen|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151894": {
+      "content": "<|speak|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151895": {
+      "content": "<|/speak|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151896": {
+      "content": "<|audio|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151897": {
+      "content": "<|/audio|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151898": {
+      "content": "<|emotion|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151899": {
+      "content": "<|/emotion|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151900": {
+      "content": "<|emotion:neutral|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151901": {
+      "content": "<|emotion:happy|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151902": {
+      "content": "<|emotion:sad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151903": {
+      "content": "<|emotion:angry|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151904": {
+      "content": "<|emotion:surprised|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151905": {
+      "content": "<|emotion:fearful|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151906": {
+      "content": "<|emotion:disgusted|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151907": {
+      "content": "<|emotion:calm|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151908": {
+      "content": "<|emotion:excited|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151909": {
+      "content": "<|emotion:curious|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151910": {
+      "content": "<|emotion:sympathetic|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151911": {
+      "content": "<|emotion:confident|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151912": {
+      "content": "<|prosody|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151913": {
+      "content": "<|/prosody|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151914": {
+      "content": "<|prosody:fast|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151915": {
+      "content": "<|prosody:slow|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151916": {
+      "content": "<|prosody:normal_speed|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151917": {
+      "content": "<|prosody:loud|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151918": {
+      "content": "<|prosody:soft|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151919": {
+      "content": "<|prosody:whisper|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151920": {
+      "content": "<|prosody:normal_volume|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151921": {
+      "content": "<|prosody:high_pitch|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151922": {
+      "content": "<|prosody:low_pitch|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151923": {
+      "content": "<|prosody:normal_pitch|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151924": {
+      "content": "<|prosody:emphasis|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151925": {
+      "content": "<|prosody:pause_short|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151926": {
+      "content": "<|prosody:pause_long|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151927": {
+      "content": "<|sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151928": {
+      "content": "<|nl|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151929": {
+      "content": "<|mask|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|bos|>",
+    "<|eos|>",
+    "<|pad|>",
+    "<|prompt|>",
+    "<|/prompt|>",
+    "<|text|>",
+    "<|/text|>",
+    "<|response|>",
+    "<|/response|>",
+    "<|system|>",
+    "<|/system|>",
+    "<|user|>",
+    "<|/user|>",
+    "<|assistant|>",
+    "<|/assistant|>",
+    "<|fim_prefix|>",
+    "<|fim_middle|>",
+    "<|fim_suffix|>",
+    "<|fim_pad|>",
+    "<|commit_before|>",
+    "<|/commit_before|>",
+    "<|commit_after|>",
+    "<|/commit_after|>",
+    "<|commit_msg|>",
+    "<|/commit_msg|>",
+    "<|diff|>",
+    "<|/diff|>",
+    "<|diff_add|>",
+    "<|diff_del|>",
+    "<|diff_ctx|>",
+    "<|reponame|>",
+    "<|/reponame|>",
+    "<|gh_stars|>",
+    "<|/gh_stars|>",
+    "<|branch|>",
+    "<|/branch|>",
+    "<|issue|>",
+    "<|/issue|>",
+    "<|issue_title|>",
+    "<|/issue_title|>",
+    "<|issue_body|>",
+    "<|/issue_body|>",
+    "<|issue_closed|>",
+    "<|issue_open|>",
+    "<|pr|>",
+    "<|/pr|>",
+    "<|pr_title|>",
+    "<|/pr_title|>",
+    "<|pr_body|>",
+    "<|/pr_body|>",
+    "<|pr_merged|>",
+    "<|pr_closed|>",
+    "<|jupyter|>",
+    "<|/jupyter|>",
+    "<|jupyter_code|>",
+    "<|/jupyter_code|>",
+    "<|jupyter_output|>",
+    "<|/jupyter_output|>",
+    "<|jupyter_text|>",
+    "<|/jupyter_text|>",
+    "<|jupyter_error|>",
+    "<|/jupyter_error|>",
+    "<|empty_output|>",
+    "<|exec|>",
+    "<|/exec|>",
+    "<|exec_result|>",
+    "<|/exec_result|>",
+    "<|exec_error|>",
+    "<|/exec_error|>",
+    "<|exec_timeout|>",
+    "<|add_file|>",
+    "<|/add_file|>",
+    "<|delete_file|>",
+    "<|/delete_file|>",
+    "<|rename_file|>",
+    "<|/rename_file|>",
+    "<|edit_file|>",
+    "<|/edit_file|>",
+    "<|read_file|>",
+    "<|/read_file|>",
+    "<|file_sep|>",
+    "<|file_content|>",
+    "<|/file_content|>",
+    "<|edit_range|>",
+    "<|/edit_range|>",
+    "<|line|>",
+    "<|/line|>",
+    "<|insert_before|>",
+    "<|insert_after|>",
+    "<|replace|>",
+    "<|/replace|>",
+    "[e~[",
+    "]~e]",
+    "[d~[",
+    "]~d]",
+    "[p~[",
+    "]~p]",
+    "]~b]",
+    "]~!b[",
+    "[m~[",
+    "]~m]",
+    "<|doc|>",
+    "<|/doc|>",
+    "<|eod|>",
+    "<|file:txt|>",
+    "<|/file:txt|>",
+    "<|file:md|>",
+    "<|/file:md|>",
+    "<|file:json|>",
+    "<|/file:json|>",
+    "<|file:xml|>",
+    "<|/file:xml|>",
+    "<|file:yaml|>",
+    "<|/file:yaml|>",
+    "<|file:html|>",
+    "<|/file:html|>",
+    "<|file:css|>",
+    "<|/file:css|>",
+    "<|file:csv|>",
+    "<|/file:csv|>",
+    "<|file:toml|>",
+    "<|/file:toml|>",
+    "<|file:ini|>",
+    "<|/file:ini|>",
+    "<|file:log|>",
+    "<|/file:log|>",
+    "<|filename|>",
+    "<|/filename|>",
+    "<|filepath|>",
+    "<|/filepath|>",
+    "<|image|>",
+    "<|/image|>",
+    "<|video|>",
+    "<|/video|>",
+    "<|gen_image|>",
+    "<|/gen_image|>",
+    "<|gen_video|>",
+    "<|/gen_video|>",
+    "<|tool_call|>",
+    "<|/tool_call|>",
+    "<|tool_result|>",
+    "<|/tool_result|>",
+    "<|function_name|>",
+    "<|/function_name|>",
+    "<|function_args|>",
+    "<|/function_args|>",
+    "<|arg_name|>",
+    "<|/arg_name|>",
+    "<|arg_value|>",
+    "<|/arg_value|>",
+    "<|tools|>",
+    "<|/tools|>",
+    "<|function_def|>",
+    "<|/function_def|>",
+    "<|tool_error|>",
+    "<|/tool_error|>",
+    "<|tool_success|>",
+    "<|tool_calls|>",
+    "<|/tool_calls|>",
+    "<|tool_id|>",
+    "<|/tool_id|>",
+    "<|code|>",
+    "<|/code|>",
+    "<|lang:python|>",
+    "<|lang:javascript|>",
+    "<|lang:typescript|>",
+    "<|lang:java|>",
+    "<|lang:cpp|>",
+    "<|lang:c|>",
+    "<|lang:csharp|>",
+    "<|lang:go|>",
+    "<|lang:rust|>",
+    "<|lang:ruby|>",
+    "<|lang:php|>",
+    "<|lang:swift|>",
+    "<|lang:kotlin|>",
+    "<|lang:scala|>",
+    "<|lang:shell|>",
+    "<|lang:bash|>",
+    "<|lang:sql|>",
+    "<|lang:r|>",
+    "<|lang:matlab|>",
+    "<|lang:lua|>",
+    "<|lang:perl|>",
+    "<|lang:haskell|>",
+    "<|lang:other|>",
+    "<|think|>",
+    "<|/think|>",
+    "<|observation|>",
+    "<|/observation|>",
+    "<|note|>",
+    "<|/note|>",
+    "<|step|>",
+    "<|/step|>",
+    "<|reflection|>",
+    "<|/reflection|>",
+    "<|hypothesis|>",
+    "<|/hypothesis|>",
+    "<|conclusion|>",
+    "<|/conclusion|>",
+    "<|confidence:high|>",
+    "<|confidence:medium|>",
+    "<|confidence:low|>",
+    "<|confidence:uncertain|>",
+    "<|uncertain|>",
+    "<|/uncertain|>",
+    "<|unknown|>",
+    "<|need_verify|>",
+    "<|may_outdated|>",
+    "<|verify|>",
+    "<|/verify|>",
+    "<|fact_check|>",
+    "<|self_correct|>",
+    "<|correction|>",
+    "<|/correction|>",
+    "<|cite|>",
+    "<|/cite|>",
+    "<|source|>",
+    "<|/source|>",
+    "<|ref|>",
+    "<|/ref|>",
+    "<|quote|>",
+    "<|/quote|>",
+    "<|context|>",
+    "<|/context|>",
+    "<|retrieved|>",
+    "<|/retrieved|>",
+    "<|grounded|>",
+    "<|ungrounded|>",
+    "<|knowledge_cutoff|>",
+    "<|beyond_knowledge|>",
+    "<|within_knowledge|>",
+    "<|listen|>",
+    "<|/listen|>",
+    "<|speak|>",
+    "<|/speak|>",
+    "<|audio|>",
+    "<|/audio|>",
+    "<|emotion|>",
+    "<|/emotion|>",
+    "<|emotion:neutral|>",
+    "<|emotion:happy|>",
+    "<|emotion:sad|>",
+    "<|emotion:angry|>",
+    "<|emotion:surprised|>",
+    "<|emotion:fearful|>",
+    "<|emotion:disgusted|>",
+    "<|emotion:calm|>",
+    "<|emotion:excited|>",
+    "<|emotion:curious|>",
+    "<|emotion:sympathetic|>",
+    "<|emotion:confident|>",
+    "<|prosody|>",
+    "<|/prosody|>",
+    "<|prosody:fast|>",
+    "<|prosody:slow|>",
+    "<|prosody:normal_speed|>",
+    "<|prosody:loud|>",
+    "<|prosody:soft|>",
+    "<|prosody:whisper|>",
+    "<|prosody:normal_volume|>",
+    "<|prosody:high_pitch|>",
+    "<|prosody:low_pitch|>",
+    "<|prosody:normal_pitch|>",
+    "<|prosody:emphasis|>",
+    "<|prosody:pause_short|>",
+    "<|prosody:pause_long|>",
+    "<|sep|>",
+    "<|nl|>",
+    "<|mask|>"
+  ],
+  "bos_token": "<|bos|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|eos|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 131072,
+  "pad_token": "<|pad|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

training_state.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:34c91c0a4284added2a08b09013db6e8311c2ab38296db54892b0c48f6a745b6
+size 3877508893