Add tiny Xiaomi-Lens-0 processor with original processor settings

README.md

@@ -0,0 +1,22 @@
+---
+library_name: transformers
+tags:
+- testing
+- image-text-to-text
+---
+# Tiny Xiaomi-Lens-0 processor
+
+Processor-only test fixture using the tiny BPE tokenizer from
+[hf-internal-testing/tiny-processor-qwen3_vl](https://huggingface.co/hf-internal-testing/tiny-processor-qwen3_vl/tree/8629850bf551ef61426c577885421520a1eacee7)
+and the processor configuration, tokenizer settings, and chat template from
+[guarin/Xiaomi-Lens-0](https://huggingface.co/guarin/Xiaomi-Lens-0/tree/1fe02dea90bfaaaf1d1311dc187a4466fdb9a1b8).
+
+Xiaomi control tokens are added to the tiny vocabulary. Token IDs are local to this
+fixture and are not compatible with the full model weights. Image and video settings
+are unchanged from Xiaomi-Lens-0; only the vocabulary is tiny. No model weights are included.
+Requires a Transformers version supporting XiaomiLens0Processor.
+
+```python
+from transformers import AutoProcessor
+processor = AutoProcessor.from_pretrained("hf-internal-testing/tiny-processor-xiaomi_lens0")
+```

chat_template.jinja

@@ -0,0 +1,154 @@
+{%- set image_count = namespace(value=0) %}
+{%- set video_count = namespace(value=0) %}
+{%- macro render_content(content, do_vision_count, is_system_content=false) %}
+    {%- if content is string %}
+        {{- content }}
+    {%- elif content is iterable and content is not mapping %}
+        {%- for item in content %}
+            {%- if 'image' in item or 'image_url' in item or item.type == 'image' %}
+                {%- if is_system_content %}
+                    {{- raise_exception('System message cannot contain images.') }}
+                {%- endif %}
+                {%- if do_vision_count %}
+                    {%- set image_count.value = image_count.value + 1 %}
+                {%- endif %}
+                {%- if add_vision_id %}
+                    {{- 'Picture ' ~ image_count.value ~ ': ' }}
+                {%- endif %}
+                {{- '<|vision_start|><|image_pad|><|vision_end|>' }}
+            {%- elif 'video' in item or item.type == 'video' %}
+                {%- if is_system_content %}
+                    {{- raise_exception('System message cannot contain videos.') }}
+                {%- endif %}
+                {%- if do_vision_count %}
+                    {%- set video_count.value = video_count.value + 1 %}
+                {%- endif %}
+                {%- if add_vision_id %}
+                    {{- 'Video ' ~ video_count.value ~ ': ' }}
+                {%- endif %}
+                {{- '<|vision_start|><|video_pad|><|vision_end|>' }}
+            {%- elif 'text' in item %}
+                {{- item.text }}
+            {%- else %}
+                {{- raise_exception('Unexpected item type in content.') }}
+            {%- endif %}
+        {%- endfor %}
+    {%- elif content is none or content is undefined %}
+        {{- '' }}
+    {%- else %}
+        {{- raise_exception('Unexpected content type.') }}
+    {%- endif %}
+{%- endmacro %}
+{%- if not messages %}
+    {{- raise_exception('No messages provided.') }}
+{%- endif %}
+{%- if tools and tools is iterable and tools is not mapping %}
+    {{- '<|im_start|>system\n' }}
+    {{- "# Tools\n\nYou have access to the following functions:\n\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>" }}
+    {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n<tool_call>\n<function=example_function_name>\n<parameter=example_parameter_1>\nvalue_1\n</parameter>\n<parameter=example_parameter_2>\nThis is the value for the second parameter\nthat can span\nmultiple lines\n</parameter>\n</function>\n</tool_call>\n\n<IMPORTANT>\nReminder:\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n</IMPORTANT>' }}
+    {%- if messages[0].role == 'system' %}
+        {%- set content = render_content(messages[0].content, false, true)|trim %}
+        {%- if content %}
+            {{- '\n\n' + content }}
+        {%- endif %}
+    {%- endif %}
+    {{- '<|im_end|>\n' }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {%- set content = render_content(messages[0].content, false, true)|trim %}
+        {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- if ns.multi_step_tool and message.role == "user" %}
+        {%- set content = render_content(message.content, false)|trim %}
+        {%- if not(content.startswith('<tool_response>') and content.endswith('</tool_response>')) %}
+            {%- set ns.multi_step_tool = false %}
+            {%- set ns.last_query_index = index %}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if ns.multi_step_tool %}
+    {{- raise_exception('No user query found in messages.') }}
+{%- endif %}
+{%- for message in messages %}
+    {%- set content = render_content(message.content, true)|trim %}
+    {%- if message.role == "system" %}
+        {%- if not loop.first %}
+            {{- raise_exception('System message must be at the beginning.') }}
+        {%- endif %}
+    {%- elif message.role == "user" %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {%- set reasoning_content = '' %}
+        {%- if message.reasoning_content is string %}
+            {%- set reasoning_content = message.reasoning_content %}
+        {%- else %}
+            {%- if '</think>' in content %}
+                {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
+                {%- set content = content.split('</think>')[-1].lstrip('\n') %}
+            {%- endif %}
+        {%- endif %}
+        {%- set reasoning_content = reasoning_content|trim %}
+        {%- if loop.index0 > ns.last_query_index %}
+            {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content + '\n</think>\n\n' + content }}
+        {%- else %}
+            {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- endif %}
+        {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if tool_call.function is defined %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {%- if loop.first %}
+                    {%- if content|trim %}
+                        {{- '\n\n<tool_call>\n<function=' + tool_call.name + '>\n' }}
+                    {%- else %}
+                        {{- '<tool_call>\n<function=' + tool_call.name + '>\n' }}
+                    {%- endif %}
+                {%- else %}
+                    {{- '\n<tool_call>\n<function=' + tool_call.name + '>\n' }}
+                {%- endif %}
+                {%- if tool_call.arguments is defined %}
+                    {%- for args_name, args_value in tool_call.arguments|items %}
+                        {{- '<parameter=' + args_name + '>\n' }}
+                        {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %}
+                        {{- args_value }}
+                        {{- '\n</parameter>\n' }}
+                    {%- endfor %}
+                {%- endif %}
+                {{- '</function>\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.previtem and loop.previtem.role != "tool" %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if not loop.last and loop.nextitem.role != "tool" %}
+            {{- '<|im_end|>\n' }}
+        {%- elif loop.last %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- else %}
+        {{- raise_exception('Unexpected message role.') }}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+    {%- if enable_thinking is defined and enable_thinking is false %}
+        {{- '<think>\n\n</think>\n\n' }}
+    {%- else %}
+        {{- '<think>\n' }}
+    {%- endif %}
+{%- endif %}

processor_config.json

@@ -0,0 +1,62 @@
+{
+  "image_processor": {
+    "do_convert_rgb": true,
+    "do_normalize": true,
+    "do_rescale": true,
+    "do_resize": true,
+    "image_mean": [
+      0.5,
+      0.5,
+      0.5
+    ],
+    "image_processor_type": "Qwen2VLImageProcessor",
+    "image_std": [
+      0.5,
+      0.5,
+      0.5
+    ],
+    "max_pixels": 8388608,
+    "merge_size": 2,
+    "min_pixels": 65536,
+    "patch_size": 16,
+    "resample": 3,
+    "rescale_factor": 0.00392156862745098,
+    "size": {
+      "longest_edge": 8388608,
+      "shortest_edge": 65536
+    },
+    "temporal_patch_size": 2
+  },
+  "processor_class": "XiaomiLens0Processor",
+  "video_processor": {
+    "do_convert_rgb": true,
+    "do_normalize": true,
+    "do_rescale": true,
+    "do_resize": true,
+    "do_sample_frames": true,
+    "fps": 2,
+    "image_mean": [
+      0.5,
+      0.5,
+      0.5
+    ],
+    "image_std": [
+      0.5,
+      0.5,
+      0.5
+    ],
+    "max_frames": 768,
+    "merge_size": 2,
+    "min_frames": 4,
+    "patch_size": 16,
+    "resample": 3,
+    "rescale_factor": 0.00392156862745098,
+    "return_metadata": false,
+    "size": {
+      "longest_edge": 25165824,
+      "shortest_edge": 4096
+    },
+    "temporal_patch_size": 2,
+    "video_processor_type": "Qwen3VLVideoProcessor"
+  }
+}

tokenizer.json

@@ -0,0 +1,1837 @@
+{
+  "version": "1.0",
+  "truncation": null,
+  "padding": null,
+  "added_tokens": [
+    {
+      "id": 500,
+      "content": "<|endoftext|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 501,
+      "content": "<|im_start|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 502,
+      "content": "<|im_end|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 503,
+      "content": "<|object_ref_start|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 504,
+      "content": "<|object_ref_end|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 505,
+      "content": "<|box_start|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 506,
+      "content": "<|box_end|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 507,
+      "content": "<|quad_start|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 508,
+      "content": "<|quad_end|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 509,
+      "content": "<|vision_start|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 510,
+      "content": "<|vision_end|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 511,
+      "content": "<|vision_pad|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 512,
+      "content": "<|image_pad|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 513,
+      "content": "<|video_pad|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 514,
+      "content": "<tool_call>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": false
+    },
+    {
+      "id": 515,
+      "content": "</tool_call>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": false
+    },
+    {
+      "id": 516,
+      "content": "<|fim_prefix|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": false
+    },
+    {
+      "id": 517,
+      "content": "<|fim_middle|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": false
+    },
+    {
+      "id": 518,
+      "content": "<|fim_suffix|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": false
+    },
+    {
+      "id": 519,
+      "content": "<|fim_pad|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": false
+    },
+    {
+      "id": 520,
+      "content": "<|repo_name|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": false
+    },
+    {
+      "id": 521,
+      "content": "<|file_sep|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": false
+    },
+    {
+      "id": 522,
+      "content": "<tool_response>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": false
+    },
+    {
+      "id": 523,
+      "content": "</tool_response>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": false
+    },
+    {
+      "id": 524,
+      "content": "<think>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 525,
+      "content": "</think>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 526,
+      "content": "<|audio_start|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 527,
+      "content": "<|audio_end|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 528,
+      "content": "<|audio_pad|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 529,
+      "content": "<|mask|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 530,
+      "content": "<bbox>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 531,
+      "content": "</bbox>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 532,
+      "content": "<mask>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 533,
+      "content": "</mask>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    }
+  ],
+  "normalizer": {
+    "type": "NFC"
+  },
+  "pre_tokenizer": {
+    "type": "Sequence",
+    "pretokenizers": [
+      {
+        "type": "Split",
+        "pattern": {
+          "Regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+"
+        },
+        "behavior": "Isolated",
+        "invert": false
+      },
+      {
+        "type": "ByteLevel",
+        "add_prefix_space": false,
+        "trim_offsets": false,
+        "use_regex": false
+      }
+    ]
+  },
+  "post_processor": {
+    "type": "ByteLevel",
+    "add_prefix_space": false,
+    "trim_offsets": false,
+    "use_regex": false
+  },
+  "decoder": {
+    "type": "ByteLevel",
+    "add_prefix_space": false,
+    "trim_offsets": false,
+    "use_regex": false
+  },
+  "model": {
+    "type": "BPE",
+    "dropout": null,
+    "unk_token": null,
+    "continuing_subword_prefix": "",
+    "end_of_word_suffix": "",
+    "fuse_unk": false,
+    "byte_fallback": false,
+    "ignore_merges": false,
+    "vocab": {
+      "!": 0,
+      "\"": 1,
+      "#": 2,
+      "$": 3,
+      "%": 4,
+      "&": 5,
+      "'": 6,
+      "(": 7,
+      ")": 8,
+      "*": 9,
+      "+": 10,
+      ",": 11,
+      "-": 12,
+      ".": 13,
+      "/": 14,
+      "0": 15,
+      "1": 16,
+      "2": 17,
+      "3": 18,
+      "4": 19,
+      "5": 20,
+      "6": 21,
+      "7": 22,
+      "8": 23,
+      "9": 24,
+      ":": 25,
+      ";": 26,
+      "<": 27,
+      "=": 28,
+      ">": 29,
+      "?": 30,
+      "@": 31,
+      "A": 32,
+      "B": 33,
+      "C": 34,
+      "D": 35,
+      "E": 36,
+      "F": 37,
+      "G": 38,
+      "H": 39,
+      "I": 40,
+      "J": 41,
+      "K": 42,
+      "L": 43,
+      "M": 44,
+      "N": 45,
+      "O": 46,
+      "P": 47,
+      "Q": 48,
+      "R": 49,
+      "S": 50,
+      "T": 51,
+      "U": 52,
+      "V": 53,
+      "W": 54,
+      "X": 55,
+      "Y": 56,
+      "Z": 57,
+      "[": 58,
+      "\\": 59,
+      "]": 60,
+      "^": 61,
+      "_": 62,
+      "`": 63,
+      "a": 64,
+      "b": 65,
+      "c": 66,
+      "d": 67,
+      "e": 68,
+      "f": 69,
+      "g": 70,
+      "h": 71,
+      "i": 72,
+      "j": 73,
+      "k": 74,
+      "l": 75,
+      "m": 76,
+      "n": 77,
+      "o": 78,
+      "p": 79,
+      "q": 80,
+      "r": 81,
+      "s": 82,
+      "t": 83,
+      "u": 84,
+      "v": 85,
+      "w": 86,
+      "x": 87,
+      "y": 88,
+      "z": 89,
+      "{": 90,
+      "|": 91,
+      "}": 92,
+      "~": 93,
+      "¡": 94,
+      "¢": 95,
+      "£": 96,
+      "¤": 97,
+      "¥": 98,
+      "¦": 99,
+      "§": 100,
+      "¨": 101,
+      "©": 102,
+      "ª": 103,
+      "«": 104,
+      "¬": 105,
+      "®": 106,
+      "¯": 107,
+      "°": 108,
+      "±": 109,
+      "²": 110,
+      "³": 111,
+      "´": 112,
+      "µ": 113,
+      "¶": 114,
+      "·": 115,
+      "¸": 116,
+      "¹": 117,
+      "º": 118,
+      "»": 119,
+      "¼": 120,
+      "½": 121,
+      "¾": 122,
+      "¿": 123,
+      "À": 124,
+      "Á": 125,
+      "Â": 126,
+      "Ã": 127,
+      "Ä": 128,
+      "Å": 129,
+      "Æ": 130,
+      "Ç": 131,
+      "È": 132,
+      "É": 133,
+      "Ê": 134,
+      "Ë": 135,
+      "Ì": 136,
+      "Í": 137,
+      "Î": 138,
+      "Ï": 139,
+      "Ð": 140,
+      "Ñ": 141,
+      "Ò": 142,
+      "Ó": 143,
+      "Ô": 144,
+      "Õ": 145,
+      "Ö": 146,
+      "×": 147,
+      "Ø": 148,
+      "Ù": 149,
+      "Ú": 150,
+      "Û": 151,
+      "Ü": 152,
+      "Ý": 153,
+      "Þ": 154,
+      "ß": 155,
+      "à": 156,
+      "á": 157,
+      "â": 158,
+      "ã": 159,
+      "ä": 160,
+      "å": 161,
+      "æ": 162,
+      "ç": 163,
+      "è": 164,
+      "é": 165,
+      "ê": 166,
+      "ë": 167,
+      "ì": 168,
+      "í": 169,
+      "î": 170,
+      "ï": 171,
+      "ð": 172,
+      "ñ": 173,
+      "ò": 174,
+      "ó": 175,
+      "ô": 176,
+      "õ": 177,
+      "ö": 178,
+      "÷": 179,
+      "ø": 180,
+      "ù": 181,
+      "ú": 182,
+      "û": 183,
+      "ü": 184,
+      "ý": 185,
+      "þ": 186,
+      "ÿ": 187,
+      "Ā": 188,
+      "ā": 189,
+      "Ă": 190,
+      "ă": 191,
+      "Ą": 192,
+      "ą": 193,
+      "Ć": 194,
+      "ć": 195,
+      "Ĉ": 196,
+      "ĉ": 197,
+      "Ċ": 198,
+      "ċ": 199,
+      "Č": 200,
+      "č": 201,
+      "Ď": 202,
+      "ď": 203,
+      "Đ": 204,
+      "đ": 205,
+      "Ē": 206,
+      "ē": 207,
+      "Ĕ": 208,
+      "ĕ": 209,
+      "Ė": 210,
+      "ė": 211,
+      "Ę": 212,
+      "ę": 213,
+      "Ě": 214,
+      "ě": 215,
+      "Ĝ": 216,
+      "ĝ": 217,
+      "Ğ": 218,
+      "ğ": 219,
+      "Ġ": 220,
+      "ġ": 221,
+      "Ģ": 222,
+      "ģ": 223,
+      "Ĥ": 224,
+      "ĥ": 225,
+      "Ħ": 226,
+      "ħ": 227,
+      "Ĩ": 228,
+      "ĩ": 229,
+      "Ī": 230,
+      "ī": 231,
+      "Ĭ": 232,
+      "ĭ": 233,
+      "Į": 234,
+      "į": 235,
+      "İ": 236,
+      "ı": 237,
+      "IJ": 238,
+      "ij": 239,
+      "Ĵ": 240,
+      "ĵ": 241,
+      "Ķ": 242,
+      "ķ": 243,
+      "ĸ": 244,
+      "Ĺ": 245,
+      "ĺ": 246,
+      "Ļ": 247,
+      "ļ": 248,
+      "Ľ": 249,
+      "ľ": 250,
+      "Ŀ": 251,
+      "ŀ": 252,
+      "Ł": 253,
+      "ł": 254,
+      "Ń": 255,
+      "ĠĠ": 256,
+      "ĠĠĠĠ": 257,
+      "in": 258,
+      "Ġt": 259,
+      "ĠĠĠĠĠĠĠĠ": 260,
+      "er": 261,
+      "ĠĠĠ": 262,
+      "on": 263,
+      "Ġa": 264,
+      "re": 265,
+      "at": 266,
+      "st": 267,
+      "en": 268,
+      "or": 269,
+      "Ġth": 270,
+      "ĊĊ": 271,
+      "Ġc": 272,
+      "le": 273,
+      "Ġs": 274,
+      "it": 275,
+      "an": 276,
+      "ar": 277,
+      "al": 278,
+      "Ġthe": 279,
+      ";Ċ": 280,
+      "Ġp": 281,
+      "Ġf": 282,
+      "ou": 283,
+      "Ġ=": 284,
+      "is": 285,
+      "ĠĠĠĠĠĠĠ": 286,
+      "ing": 287,
+      "es": 288,
+      "Ġw": 289,
+      "ion": 290,
+      "ed": 291,
+      "ic": 292,
+      "Ġb": 293,
+      "Ġd": 294,
+      "et": 295,
+      "Ġm": 296,
+      "Ġo": 297,
+      "ĉĉ": 298,
+      "ro": 299,
+      "as": 300,
+      "el": 301,
+      "ct": 302,
+      "nd": 303,
+      "Ġin": 304,
+      "Ġh": 305,
+      "ent": 306,
+      "id": 307,
+      "Ġn": 308,
+      "am": 309,
+      "ĠĠĠĠĠĠĠĠĠĠĠ": 310,
+      "Ġto": 311,
+      "Ġre": 312,
+      "--": 313,
+      "Ġ{": 314,
+      "Ġof": 315,
+      "om": 316,
+      ");Ċ": 317,
+      "im": 318,
+      "čĊ": 319,
+      "Ġ(": 320,
+      "il": 321,
+      "//": 322,
+      "Ġand": 323,
+      "ur": 324,
+      "se": 325,
+      "Ġl": 326,
+      "ex": 327,
+      "ĠS": 328,
+      "ad": 329,
+      "Ġ\"": 330,
+      "ch": 331,
+      "ut": 332,
+      "if": 333,
+      "**": 334,
+      "Ġ}": 335,
+      "em": 336,
+      "ol": 337,
+      "ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ": 338,
+      "th": 339,
+      ")Ċ": 340,
+      "Ġ{Ċ": 341,
+      "Ġg": 342,
+      "ig": 343,
+      "iv": 344,
+      ",Ċ": 345,
+      "ce": 346,
+      "od": 347,
+      "Ġv": 348,
+      "ate": 349,
+      "ĠT": 350,
+      "ag": 351,
+      "ay": 352,
+      "Ġ*": 353,
+      "ot": 354,
+      "us": 355,
+      "ĠC": 356,
+      "Ġst": 357,
+      "ĠI": 358,
+      "un": 359,
+      "ul": 360,
+      "ue": 361,
+      "ĠA": 362,
+      "ow": 363,
+      "Ġ'": 364,
+      "ew": 365,
+      "Ġ<": 366,
+      "ation": 367,
+      "()": 368,
+      "Ġfor": 369,
+      "ab": 370,
+      "ort": 371,
+      "um": 372,
+      "ame": 373,
+      "Ġis": 374,
+      "pe": 375,
+      "tr": 376,
+      "ck": 377,
+      "âĢ": 378,
+      "Ġy": 379,
+      "ist": 380,
+      "----": 381,
+      ".ĊĊ": 382,
+      "he": 383,
+      "Ġe": 384,
+      "lo": 385,
+      "ĠM": 386,
+      "Ġbe": 387,
+      "ers": 388,
+      "Ġon": 389,
+      "Ġcon": 390,
+      "ap": 391,
+      "ub": 392,
+      "ĠP": 393,
+      "ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ": 394,
+      "ass": 395,
+      "int": 396,
+      ">Ċ": 397,
+      "ly": 398,
+      "urn": 399,
+      "Ġ$": 400,
+      ";ĊĊ": 401,
+      "av": 402,
+      "port": 403,
+      "ir": 404,
+      "->": 405,
+      "nt": 406,
+      "ction": 407,
+      "end": 408,
+      "Ġde": 409,
+      "ith": 410,
+      "out": 411,
+      "turn": 412,
+      "our": 413,
+      "ĠĠĠĠĠ": 414,
+      "lic": 415,
+      "res": 416,
+      "pt": 417,
+      "==": 418,
+      "Ġthis": 419,
+      "Ġwh": 420,
+      "Ġif": 421,
+      "ĠD": 422,
+      "ver": 423,
+      "age": 424,
+      "ĠB": 425,
+      "ht": 426,
+      "ext": 427,
+      "=\"": 428,
+      "Ġthat": 429,
+      "****": 430,
+      "ĠR": 431,
+      "Ġit": 432,
+      "ess": 433,
+      "ĠF": 434,
+      "Ġr": 435,
+      "os": 436,
+      "and": 437,
+      "Ġas": 438,
+      "ect": 439,
+      "ke": 440,
+      "rom": 441,
+      "Ġ//": 442,
+      "con": 443,
+      "ĠL": 444,
+      "(\"": 445,
+      "qu": 446,
+      "lass": 447,
+      "Ġwith": 448,
+      "iz": 449,
+      "de": 450,
+      "ĠN": 451,
+      "Ġal": 452,
+      "op": 453,
+      "up": 454,
+      "get": 455,
+      "Ġ}Ċ": 456,
+      "ile": 457,
+      "Ġan": 458,
+      "ata": 459,
+      "ore": 460,
+      "ri": 461,
+      "Ġpro": 462,
+      ";čĊ": 463,
+      "ĉĉĉĉ": 464,
+      "ter": 465,
+      "ain": 466,
+      "ĠW": 467,
+      "ĠE": 468,
+      "Ġcom": 469,
+      "Ġreturn": 470,
+      "art": 471,
+      "ĠH": 472,
+      "ack": 473,
+      "import": 474,
+      "ublic": 475,
+      "Ġor": 476,
+      "est": 477,
+      "ment": 478,
+      "ĠG": 479,
+      "able": 480,
+      "Ġ-": 481,
+      "ine": 482,
+      "ill": 483,
+      "ind": 484,
+      "ere": 485,
+      "::": 486,
+      "ity": 487,
+      "Ġ+": 488,
+      "Ġtr": 489,
+      "elf": 490,
+      "ight": 491,
+      "('": 492,
+      "orm": 493,
+      "ult": 494,
+      "str": 495,
+      "..": 496,
+      "\",": 497,
+      "Ġyou": 498,
+      "ype": 499
+    },
+    "merges": [
+      [
+        "Ġ",
+        "Ġ"
+      ],
+      [
+        "ĠĠ",
+        "ĠĠ"
+      ],
+      [
+        "i",
+        "n"
+      ],
+      [
+        "Ġ",
+        "t"
+      ],
+      [
+        "ĠĠĠĠ",
+        "ĠĠĠĠ"
+      ],
+      [
+        "e",
+        "r"
+      ],
+      [
+        "ĠĠ",
+        "Ġ"
+      ],
+      [
+        "o",
+        "n"
+      ],
+      [
+        "Ġ",
+        "a"
+      ],
+      [
+        "r",
+        "e"
+      ],
+      [
+        "a",
+        "t"
+      ],
+      [
+        "s",
+        "t"
+      ],
+      [
+        "e",
+        "n"
+      ],
+      [
+        "o",
+        "r"
+      ],
+      [
+        "Ġt",
+        "h"
+      ],
+      [
+        "Ċ",
+        "Ċ"
+      ],
+      [
+        "Ġ",
+        "c"
+      ],
+      [
+        "l",
+        "e"
+      ],
+      [
+        "Ġ",
+        "s"
+      ],
+      [
+        "i",
+        "t"
+      ],
+      [
+        "a",
+        "n"
+      ],
+      [
+        "a",
+        "r"
+      ],
+      [
+        "a",
+        "l"
+      ],
+      [
+        "Ġth",
+        "e"
+      ],
+      [
+        ";",
+        "Ċ"
+      ],
+      [
+        "Ġ",
+        "p"
+      ],
+      [
+        "Ġ",
+        "f"
+      ],
+      [
+        "o",
+        "u"
+      ],
+      [
+        "Ġ",
+        "="
+      ],
+      [
+        "i",
+        "s"
+      ],
+      [
+        "ĠĠĠĠ",
+        "ĠĠĠ"
+      ],
+      [
+        "in",
+        "g"
+      ],
+      [
+        "e",
+        "s"
+      ],
+      [
+        "Ġ",
+        "w"
+      ],
+      [
+        "i",
+        "on"
+      ],
+      [
+        "e",
+        "d"
+      ],
+      [
+        "i",
+        "c"
+      ],
+      [
+        "Ġ",
+        "b"
+      ],
+      [
+        "Ġ",
+        "d"
+      ],
+      [
+        "e",
+        "t"
+      ],
+      [
+        "Ġ",
+        "m"
+      ],
+      [
+        "Ġ",
+        "o"
+      ],
+      [
+        "ĉ",
+        "ĉ"
+      ],
+      [
+        "r",
+        "o"
+      ],
+      [
+        "a",
+        "s"
+      ],
+      [
+        "e",
+        "l"
+      ],
+      [
+        "c",
+        "t"
+      ],
+      [
+        "n",
+        "d"
+      ],
+      [
+        "Ġ",
+        "in"
+      ],
+      [
+        "Ġ",
+        "h"
+      ],
+      [
+        "en",
+        "t"
+      ],
+      [
+        "i",
+        "d"
+      ],
+      [
+        "Ġ",
+        "n"
+      ],
+      [
+        "a",
+        "m"
+      ],
+      [
+        "ĠĠĠĠĠĠĠĠ",
+        "ĠĠĠ"
+      ],
+      [
+        "Ġt",
+        "o"
+      ],
+      [
+        "Ġ",
+        "re"
+      ],
+      [
+        "-",
+        "-"
+      ],
+      [
+        "Ġ",
+        "{"
+      ],
+      [
+        "Ġo",
+        "f"
+      ],
+      [
+        "o",
+        "m"
+      ],
+      [
+        ")",
+        ";Ċ"
+      ],
+      [
+        "i",
+        "m"
+      ],
+      [
+        "č",
+        "Ċ"
+      ],
+      [
+        "Ġ",
+        "("
+      ],
+      [
+        "i",
+        "l"
+      ],
+      [
+        "/",
+        "/"
+      ],
+      [
+        "Ġa",
+        "nd"
+      ],
+      [
+        "u",
+        "r"
+      ],
+      [
+        "s",
+        "e"
+      ],
+      [
+        "Ġ",
+        "l"
+      ],
+      [
+        "e",
+        "x"
+      ],
+      [
+        "Ġ",
+        "S"
+      ],
+      [
+        "a",
+        "d"
+      ],
+      [
+        "Ġ",
+        "\""
+      ],
+      [
+        "c",
+        "h"
+      ],
+      [
+        "u",
+        "t"
+      ],
+      [
+        "i",
+        "f"
+      ],
+      [
+        "*",
+        "*"
+      ],
+      [
+        "Ġ",
+        "}"
+      ],
+      [
+        "e",
+        "m"
+      ],
+      [
+        "o",
+        "l"
+      ],
+      [
+        "ĠĠĠĠĠĠĠĠ",
+        "ĠĠĠĠĠĠĠĠ"
+      ],
+      [
+        "t",
+        "h"
+      ],
+      [
+        ")",
+        "Ċ"
+      ],
+      [
+        "Ġ{",
+        "Ċ"
+      ],
+      [
+        "Ġ",
+        "g"
+      ],
+      [
+        "i",
+        "g"
+      ],
+      [
+        "i",
+        "v"
+      ],
+      [
+        ",",
+        "Ċ"
+      ],
+      [
+        "c",
+        "e"
+      ],
+      [
+        "o",
+        "d"
+      ],
+      [
+        "Ġ",
+        "v"
+      ],
+      [
+        "at",
+        "e"
+      ],
+      [
+        "Ġ",
+        "T"
+      ],
+      [
+        "a",
+        "g"
+      ],
+      [
+        "a",
+        "y"
+      ],
+      [
+        "Ġ",
+        "*"
+      ],
+      [
+        "o",
+        "t"
+      ],
+      [
+        "u",
+        "s"
+      ],
+      [
+        "Ġ",
+        "C"
+      ],
+      [
+        "Ġ",
+        "st"
+      ],
+      [
+        "Ġ",
+        "I"
+      ],
+      [
+        "u",
+        "n"
+      ],
+      [
+        "u",
+        "l"
+      ],
+      [
+        "u",
+        "e"
+      ],
+      [
+        "Ġ",
+        "A"
+      ],
+      [
+        "o",
+        "w"
+      ],
+      [
+        "Ġ",
+        "'"
+      ],
+      [
+        "e",
+        "w"
+      ],
+      [
+        "Ġ",
+        "<"
+      ],
+      [
+        "at",
+        "ion"
+      ],
+      [
+        "(",
+        ")"
+      ],
+      [
+        "Ġf",
+        "or"
+      ],
+      [
+        "a",
+        "b"
+      ],
+      [
+        "or",
+        "t"
+      ],
+      [
+        "u",
+        "m"
+      ],
+      [
+        "am",
+        "e"
+      ],
+      [
+        "Ġ",
+        "is"
+      ],
+      [
+        "p",
+        "e"
+      ],
+      [
+        "t",
+        "r"
+      ],
+      [
+        "c",
+        "k"
+      ],
+      [
+        "â",
+        "Ģ"
+      ],
+      [
+        "Ġ",
+        "y"
+      ],
+      [
+        "i",
+        "st"
+      ],
+      [
+        "--",
+        "--"
+      ],
+      [
+        ".",
+        "ĊĊ"
+      ],
+      [
+        "h",
+        "e"
+      ],
+      [
+        "Ġ",
+        "e"
+      ],
+      [
+        "l",
+        "o"
+      ],
+      [
+        "Ġ",
+        "M"
+      ],
+      [
+        "Ġb",
+        "e"
+      ],
+      [
+        "er",
+        "s"
+      ],
+      [
+        "Ġ",
+        "on"
+      ],
+      [
+        "Ġc",
+        "on"
+      ],
+      [
+        "a",
+        "p"
+      ],
+      [
+        "u",
+        "b"
+      ],
+      [
+        "Ġ",
+        "P"
+      ],
+      [
+        "ĠĠĠĠĠĠĠĠ",
+        "ĠĠĠĠĠĠĠ"
+      ],
+      [
+        "as",
+        "s"
+      ],
+      [
+        "in",
+        "t"
+      ],
+      [
+        ">",
+        "Ċ"
+      ],
+      [
+        "l",
+        "y"
+      ],
+      [
+        "ur",
+        "n"
+      ],
+      [
+        "Ġ",
+        "$"
+      ],
+      [
+        ";",
+        "ĊĊ"
+      ],
+      [
+        "a",
+        "v"
+      ],
+      [
+        "p",
+        "ort"
+      ],
+      [
+        "i",
+        "r"
+      ],
+      [
+        "-",
+        ">"
+      ],
+      [
+        "n",
+        "t"
+      ],
+      [
+        "ct",
+        "ion"
+      ],
+      [
+        "en",
+        "d"
+      ],
+      [
+        "Ġd",
+        "e"
+      ],
+      [
+        "it",
+        "h"
+      ],
+      [
+        "ou",
+        "t"
+      ],
+      [
+        "t",
+        "urn"
+      ],
+      [
+        "ou",
+        "r"
+      ],
+      [
+        "ĠĠĠĠ",
+        "Ġ"
+      ],
+      [
+        "l",
+        "ic"
+      ],
+      [
+        "re",
+        "s"
+      ],
+      [
+        "p",
+        "t"
+      ],
+      [
+        "=",
+        "="
+      ],
+      [
+        "Ġth",
+        "is"
+      ],
+      [
+        "Ġw",
+        "h"
+      ],
+      [
+        "Ġ",
+        "if"
+      ],
+      [
+        "Ġ",
+        "D"
+      ],
+      [
+        "v",
+        "er"
+      ],
+      [
+        "ag",
+        "e"
+      ],
+      [
+        "Ġ",
+        "B"
+      ],
+      [
+        "h",
+        "t"
+      ],
+      [
+        "ex",
+        "t"
+      ],
+      [
+        "=",
+        "\""
+      ],
+      [
+        "Ġth",
+        "at"
+      ],
+      [
+        "**",
+        "**"
+      ],
+      [
+        "Ġ",
+        "R"
+      ],
+      [
+        "Ġ",
+        "it"
+      ],
+      [
+        "es",
+        "s"
+      ],
+      [
+        "Ġ",
+        "F"
+      ],
+      [
+        "Ġ",
+        "r"
+      ],
+      [
+        "o",
+        "s"
+      ],
+      [
+        "an",
+        "d"
+      ],
+      [
+        "Ġa",
+        "s"
+      ],
+      [
+        "e",
+        "ct"
+      ],
+      [
+        "k",
+        "e"
+      ],
+      [
+        "ro",
+        "m"
+      ],
+      [
+        "Ġ",
+        "//"
+      ],
+      [
+        "c",
+        "on"
+      ],
+      [
+        "Ġ",
+        "L"
+      ],
+      [
+        "(",
+        "\""
+      ],
+      [
+        "q",
+        "u"
+      ],
+      [
+        "l",
+        "ass"
+      ],
+      [
+        "Ġw",
+        "ith"
+      ],
+      [
+        "i",
+        "z"
+      ],
+      [
+        "d",
+        "e"
+      ],
+      [
+        "Ġ",
+        "N"
+      ],
+      [
+        "Ġa",
+        "l"
+      ],
+      [
+        "o",
+        "p"
+      ],
+      [
+        "u",
+        "p"
+      ],
+      [
+        "g",
+        "et"
+      ],
+      [
+        "Ġ}",
+        "Ċ"
+      ],
+      [
+        "i",
+        "le"
+      ],
+      [
+        "Ġa",
+        "n"
+      ],
+      [
+        "at",
+        "a"
+      ],
+      [
+        "o",
+        "re"
+      ],
+      [
+        "r",
+        "i"
+      ],
+      [
+        "Ġp",
+        "ro"
+      ],
+      [
+        ";",
+        "čĊ"
+      ],
+      [
+        "ĉĉ",
+        "ĉĉ"
+      ],
+      [
+        "t",
+        "er"
+      ],
+      [
+        "a",
+        "in"
+      ],
+      [
+        "Ġ",
+        "W"
+      ],
+      [
+        "Ġ",
+        "E"
+      ],
+      [
+        "Ġc",
+        "om"
+      ],
+      [
+        "Ġre",
+        "turn"
+      ],
+      [
+        "ar",
+        "t"
+      ],
+      [
+        "Ġ",
+        "H"
+      ],
+      [
+        "a",
+        "ck"
+      ],
+      [
+        "im",
+        "port"
+      ],
+      [
+        "ub",
+        "lic"
+      ],
+      [
+        "Ġ",
+        "or"
+      ],
+      [
+        "e",
+        "st"
+      ],
+      [
+        "m",
+        "ent"
+      ],
+      [
+        "Ġ",
+        "G"
+      ],
+      [
+        "ab",
+        "le"
+      ],
+      [
+        "Ġ",
+        "-"
+      ],
+      [
+        "in",
+        "e"
+      ],
+      [
+        "il",
+        "l"
+      ],
+      [
+        "in",
+        "d"
+      ],
+      [
+        "er",
+        "e"
+      ],
+      [
+        ":",
+        ":"
+      ],
+      [
+        "it",
+        "y"
+      ],
+      [
+        "Ġ",
+        "+"
+      ],
+      [
+        "Ġt",
+        "r"
+      ],
+      [
+        "el",
+        "f"
+      ],
+      [
+        "ig",
+        "ht"
+      ],
+      [
+        "(",
+        "'"
+      ],
+      [
+        "or",
+        "m"
+      ],
+      [
+        "ul",
+        "t"
+      ],
+      [
+        "st",
+        "r"
+      ],
+      [
+        ".",
+        "."
+      ],
+      [
+        "\"",
+        ","
+      ],
+      [
+        "Ġy",
+        "ou"
+      ],
+      [
+        "y",
+        "pe"
+      ]
+    ]
+  }
+}

tokenizer_config.json

@@ -0,0 +1,35 @@
+{
+  "add_prefix_space": false,
+  "audio_bos_token": "<|audio_start|>",
+  "audio_eos_token": "<|audio_end|>",
+  "audio_token": "<|audio_pad|>",
+  "backend": "tokenizers",
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": [
+    "<|mask|>"
+  ],
+  "image_token": "<|image_pad|>",
+  "is_local": true,
+  "model_max_length": 16384,
+  "model_specific_special_tokens": {
+    "audio_bos_token": "<|audio_start|>",
+    "audio_eos_token": "<|audio_end|>",
+    "audio_token": "<|audio_pad|>",
+    "image_token": "<|image_pad|>",
+    "video_token": "<|video_pad|>",
+    "vision_bos_token": "<|vision_start|>",
+    "vision_eos_token": "<|vision_end|>"
+  },
+  "pad_token": "<|endoftext|>",
+  "pretokenize_regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?[\\p{L}\\p{M}]+|\\p{N}| ?[^\\s\\p{L}\\p{M}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+",
+  "processor_class": "XiaomiLens0Processor",
+  "split_special_tokens": false,
+  "tokenizer_class": "TokenizersBackend",
+  "unk_token": null,
+  "video_token": "<|video_pad|>",
+  "vision_bos_token": "<|vision_start|>",
+  "vision_eos_token": "<|vision_end|>"
+}