Upload tokenizer_config_2_5.json

tokenizer_config_2_5.json

@@ -0,0 +1,28 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<|startoftext|>",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|im_end|>",
+  "image_end_token": "<|image_end|>",
+  "image_start_token": "<|image_start|>",
+  "image_thumbnail": "<|img_thumbnail|>",
+  "image_token": "<image>",
+  "is_local": true,
+  "legacy": false,
+  "model_max_length": 1000000000000000019884624838656,
+  "model_specific_special_tokens": {
+    "image_end_token": "<|image_end|>",
+    "image_start_token": "<|image_start|>",
+    "image_thumbnail": "<|img_thumbnail|>",
+    "image_token": "<image>"
+  },
+  "pad_token": "<|pad|>",
+  "processor_class": "Lfm2VlProcessor",
+  "return_token_type_ids": false,
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "TokenizersBackend",
+  "use_default_system_prompt": false,
+  "use_fast": true,
+  "chat_template": "{%- set keep_past_thinking = keep_past_thinking | default(false) -%}{%- set ns = namespace(system_prompt=\"\") -%}{%- if messages[0][\"role\"] == \"system\" -%}{%- set sys_content = messages[0][\"content\"] -%}{%- if sys_content is not string -%}{%- for item in sys_content -%}{%- if item[\"type\"] == \"text\" -%}{%- set ns.system_prompt = ns.system_prompt + item[\"text\"] -%}{%- endif -%}{%- endfor -%}{%- else -%}{%- set ns.system_prompt = sys_content -%}{%- endif -%}{%- set messages = messages[1:] -%}{%- endif -%}{%- if tools -%}{%- set ns.system_prompt = ns.system_prompt + (\"\n\" if ns.system_prompt else \"\") + \"List of tools: [\" -%}{%- for tool in tools -%}{%- if tool is not string -%}{%- set tool = tool | tojson -%}{%- endif -%}{%- set ns.system_prompt = ns.system_prompt + tool -%}{%- if not loop.last -%}{%- set ns.system_prompt = ns.system_prompt + \", \" -%}{%- endif -%}{%- endfor -%}{%- set ns.system_prompt = ns.system_prompt + \"]\" -%}{%- endif -%}{%- if ns.system_prompt -%}{{- \"<|im_start|>system\n\" + ns.system_prompt + \"<|im_end|>\n\" -}}{%- endif -%}{%- set ns.last_assistant_index = -1 -%}{%- for message in messages -%}{%- if message[\"role\"] == \"assistant\" -%}{%- set ns.last_assistant_index = loop.index0 -%}{%- endif -%}{%- endfor -%}{%- for message in messages -%}{{- \"<|im_start|>\" + message[\"role\"] + \"\n\" -}}{%- if message[\"content\"] is not string -%}{%- set ns.content = \"\" -%}{%- for item in message[\"content\"] -%}{%- if item[\"type\"] == \"image\" -%}{%- set ns.content = ns.content + \"<image>\" -%}{%- elif item[\"type\"] == \"text\" -%}{%- set ns.content = ns.content + item[\"text\"] -%}{%- else -%}{%- set ns.content = ns.content + item | tojson -%}{%- endif -%}{%- endfor -%}{%- set content = ns.content -%}{%- else -%}{%- set content = message[\"content\"] -%}{%- endif -%}{%- if message[\"role\"] == \"assistant\" and not keep_past_thinking and loop.index0 != ns.last_assistant_index -%}{%- if \"</think>\" in content -%}{%- set content = content.split(\"</think>\")[-1] | trim -%}{%- endif -%}{%- endif -%}{{- content + \"<|im_end|>\n\" -}}{%- endfor -%}{%- if add_generation_prompt -%}{{- \"<|im_start|>assistant\n\" -}}{%- endif -%}"
+}