Add tiny processor fixture for testing

LICENSE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2024 Alibaba Cloud
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

chat_template.json

@@ -0,0 +1,3 @@
+{
+    "chat_template": "{% set image_count = namespace(value=0) %}{% set video_count = namespace(value=0) %}{% for message in messages %}{% if loop.first and message['role'] != 'system' %}<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n{% endif %}<|im_start|>{{ message['role'] }}\n{% if message['content'] is string %}{{ message['content'] }}<|im_end|>\n{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}{% set image_count.value = image_count.value + 1 %}{% if add_vision_id %}Picture {{ image_count.value }}: {% endif %}<|vision_start|><|image_pad|><|vision_end|>{% elif content['type'] == 'video' or 'video' in content %}{% set video_count.value = video_count.value + 1 %}{% if add_vision_id %}Video {{ video_count.value }}: {% endif %}<|vision_start|><|video_pad|><|vision_end|>{% elif 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}<|im_end|>\n{% endif %}{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant\n{% endif %}"
+}

config.json

@@ -0,0 +1,52 @@
+{
+  "architectures": [
+    "Qwen2VLForConditionalGeneration"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 500,
+  "eos_token_id": 502,
+  "vision_start_token_id": 151652,
+  "vision_end_token_id": 151653,
+  "vision_token_id": 151654,
+  "image_token_id": 151655,
+  "video_token_id": 151656,
+  "hidden_act": "silu",
+  "hidden_size": 3584,
+  "initializer_range": 0.02,
+  "intermediate_size": 18944,
+  "max_position_embeddings": 32768,
+  "max_window_layers": 28,
+  "model_type": "qwen2_vl",
+  "num_attention_heads": 28,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 4,
+  "rms_norm_eps": 1e-06,
+  "rope_theta": 1000000.0,
+  "sliding_window": 32768,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.41.2",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "vision_config": {
+    "depth": 32,
+    "embed_dim": 1280,
+    "mlp_ratio": 4,
+    "num_heads": 16,
+    "in_chans": 3,
+    "hidden_size": 3584,
+    "patch_size": 14,
+    "spatial_merge_size": 2,
+    "spatial_patch_size": 14,
+    "temporal_patch_size": 2
+  },
+  "rope_scaling": {
+    "type": "mrope",
+    "mrope_section": [
+      16,
+      24,
+      24
+    ]
+  },
+  "vocab_size": 500
+}

generation_config.json

@@ -0,0 +1,14 @@
+{
+  "bos_token_id": 151643,
+  "pad_token_id": 151643,
+  "do_sample": true,
+  "eos_token_id": [
+    151645,
+    151643
+  ],
+  "repetition_penalty": 1.0,
+  "temperature": 0.01,
+  "top_p": 0.001,
+  "top_k": 1,
+  "transformers_version": "4.37.0"
+}

preprocessor_config.json

@@ -0,0 +1,19 @@
+{
+  "min_pixels": 3136,
+  "max_pixels": 12845056,
+  "patch_size": 14,
+  "temporal_patch_size": 2,
+  "merge_size": 2,
+  "image_mean": [
+    0.48145466,
+    0.4578275,
+    0.40821073
+  ],
+  "image_std": [
+    0.26862954,
+    0.26130258,
+    0.27577711
+  ],
+  "image_processor_type": "Qwen2VLImageProcessor",
+  "processor_class": "Qwen2VLProcessor"
+}

tokenizer.json

@@ -0,0 +1,906 @@
+{
+  "version": "1.0",
+  "truncation": null,
+  "padding": null,
+  "added_tokens": [
+    {
+      "id": 500,
+      "content": "<|endoftext|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 501,
+      "content": "<|im_start|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 502,
+      "content": "<|im_end|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 503,
+      "content": "<|object_ref_start|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 504,
+      "content": "<|object_ref_end|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 505,
+      "content": "<|box_start|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 506,
+      "content": "<|box_end|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 507,
+      "content": "<|quad_start|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 508,
+      "content": "<|quad_end|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 509,
+      "content": "<|vision_start|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 510,
+      "content": "<|vision_end|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 511,
+      "content": "<|vision_pad|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    }
+  ],
+  "normalizer": {
+    "type": "NFC"
+  },
+  "pre_tokenizer": {
+    "type": "Sequence",
+    "pretokenizers": [
+      {
+        "type": "Split",
+        "pattern": {
+          "Regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+"
+        },
+        "behavior": "Isolated",
+        "invert": false
+      },
+      {
+        "type": "ByteLevel",
+        "add_prefix_space": false,
+        "trim_offsets": false,
+        "use_regex": false
+      }
+    ]
+  },
+  "post_processor": {
+    "type": "ByteLevel",
+    "add_prefix_space": false,
+    "trim_offsets": false,
+    "use_regex": false
+  },
+  "decoder": {
+    "type": "ByteLevel",
+    "add_prefix_space": false,
+    "trim_offsets": false,
+    "use_regex": false
+  },
+  "model": {
+    "type": "BPE",
+    "dropout": null,
+    "unk_token": null,
+    "continuing_subword_prefix": "",
+    "end_of_word_suffix": "",
+    "fuse_unk": false,
+    "byte_fallback": false,
+    "vocab": {
+      "!": 0,
+      "\"": 1,
+      "#": 2,
+      "$": 3,
+      "%": 4,
+      "&": 5,
+      "'": 6,
+      "(": 7,
+      ")": 8,
+      "*": 9,
+      "+": 10,
+      ",": 11,
+      "-": 12,
+      ".": 13,
+      "/": 14,
+      "0": 15,
+      "1": 16,
+      "2": 17,
+      "3": 18,
+      "4": 19,
+      "5": 20,
+      "6": 21,
+      "7": 22,
+      "8": 23,
+      "9": 24,
+      ":": 25,
+      ";": 26,
+      "<": 27,
+      "=": 28,
+      ">": 29,
+      "?": 30,
+      "@": 31,
+      "A": 32,
+      "B": 33,
+      "C": 34,
+      "D": 35,
+      "E": 36,
+      "F": 37,
+      "G": 38,
+      "H": 39,
+      "I": 40,
+      "J": 41,
+      "K": 42,
+      "L": 43,
+      "M": 44,
+      "N": 45,
+      "O": 46,
+      "P": 47,
+      "Q": 48,
+      "R": 49,
+      "S": 50,
+      "T": 51,
+      "U": 52,
+      "V": 53,
+      "W": 54,
+      "X": 55,
+      "Y": 56,
+      "Z": 57,
+      "[": 58,
+      "\\": 59,
+      "]": 60,
+      "^": 61,
+      "_": 62,
+      "`": 63,
+      "a": 64,
+      "b": 65,
+      "c": 66,
+      "d": 67,
+      "e": 68,
+      "f": 69,
+      "g": 70,
+      "h": 71,
+      "i": 72,
+      "j": 73,
+      "k": 74,
+      "l": 75,
+      "m": 76,
+      "n": 77,
+      "o": 78,
+      "p": 79,
+      "q": 80,
+      "r": 81,
+      "s": 82,
+      "t": 83,
+      "u": 84,
+      "v": 85,
+      "w": 86,
+      "x": 87,
+      "y": 88,
+      "z": 89,
+      "{": 90,
+      "|": 91,
+      "}": 92,
+      "~": 93,
+      "¡": 94,
+      "¢": 95,
+      "£": 96,
+      "¤": 97,
+      "¥": 98,
+      "¦": 99,
+      "§": 100,
+      "¨": 101,
+      "©": 102,
+      "ª": 103,
+      "«": 104,
+      "¬": 105,
+      "®": 106,
+      "¯": 107,
+      "°": 108,
+      "±": 109,
+      "²": 110,
+      "³": 111,
+      "´": 112,
+      "µ": 113,
+      "¶": 114,
+      "·": 115,
+      "¸": 116,
+      "¹": 117,
+      "º": 118,
+      "»": 119,
+      "¼": 120,
+      "½": 121,
+      "¾": 122,
+      "¿": 123,
+      "À": 124,
+      "Á": 125,
+      "Â": 126,
+      "Ã": 127,
+      "Ä": 128,
+      "Å": 129,
+      "Æ": 130,
+      "Ç": 131,
+      "È": 132,
+      "É": 133,
+      "Ê": 134,
+      "Ë": 135,
+      "Ì": 136,
+      "Í": 137,
+      "Î": 138,
+      "Ï": 139,
+      "Ð": 140,
+      "Ñ": 141,
+      "Ò": 142,
+      "Ó": 143,
+      "Ô": 144,
+      "Õ": 145,
+      "Ö": 146,
+      "×": 147,
+      "Ø": 148,
+      "Ù": 149,
+      "Ú": 150,
+      "Û": 151,
+      "Ü": 152,
+      "Ý": 153,
+      "Þ": 154,
+      "ß": 155,
+      "à": 156,
+      "á": 157,
+      "â": 158,
+      "ã": 159,
+      "ä": 160,
+      "å": 161,
+      "æ": 162,
+      "ç": 163,
+      "è": 164,
+      "é": 165,
+      "ê": 166,
+      "ë": 167,
+      "ì": 168,
+      "í": 169,
+      "î": 170,
+      "ï": 171,
+      "ð": 172,
+      "ñ": 173,
+      "ò": 174,
+      "ó": 175,
+      "ô": 176,
+      "õ": 177,
+      "ö": 178,
+      "÷": 179,
+      "ø": 180,
+      "ù": 181,
+      "ú": 182,
+      "û": 183,
+      "ü": 184,
+      "ý": 185,
+      "þ": 186,
+      "ÿ": 187,
+      "Ā": 188,
+      "ā": 189,
+      "Ă": 190,
+      "ă": 191,
+      "Ą": 192,
+      "ą": 193,
+      "Ć": 194,
+      "ć": 195,
+      "Ĉ": 196,
+      "ĉ": 197,
+      "Ċ": 198,
+      "ċ": 199,
+      "Č": 200,
+      "č": 201,
+      "Ď": 202,
+      "ď": 203,
+      "Đ": 204,
+      "đ": 205,
+      "Ē": 206,
+      "ē": 207,
+      "Ĕ": 208,
+      "ĕ": 209,
+      "Ė": 210,
+      "ė": 211,
+      "Ę": 212,
+      "ę": 213,
+      "Ě": 214,
+      "ě": 215,
+      "Ĝ": 216,
+      "ĝ": 217,
+      "Ğ": 218,
+      "ğ": 219,
+      "Ġ": 220,
+      "ġ": 221,
+      "Ģ": 222,
+      "ģ": 223,
+      "Ĥ": 224,
+      "ĥ": 225,
+      "Ħ": 226,
+      "ħ": 227,
+      "Ĩ": 228,
+      "ĩ": 229,
+      "Ī": 230,
+      "ī": 231,
+      "Ĭ": 232,
+      "ĭ": 233,
+      "Į": 234,
+      "į": 235,
+      "İ": 236,
+      "ı": 237,
+      "IJ": 238,
+      "ij": 239,
+      "Ĵ": 240,
+      "ĵ": 241,
+      "Ķ": 242,
+      "ķ": 243,
+      "ĸ": 244,
+      "Ĺ": 245,
+      "ĺ": 246,
+      "Ļ": 247,
+      "ļ": 248,
+      "Ľ": 249,
+      "ľ": 250,
+      "Ŀ": 251,
+      "ŀ": 252,
+      "Ł": 253,
+      "ł": 254,
+      "Ń": 255,
+      "ĠĠ": 256,
+      "ĠĠĠĠ": 257,
+      "in": 258,
+      "Ġt": 259,
+      "ĠĠĠĠĠĠĠĠ": 260,
+      "er": 261,
+      "ĠĠĠ": 262,
+      "on": 263,
+      "Ġa": 264,
+      "re": 265,
+      "at": 266,
+      "st": 267,
+      "en": 268,
+      "or": 269,
+      "Ġth": 270,
+      "ĊĊ": 271,
+      "Ġc": 272,
+      "le": 273,
+      "Ġs": 274,
+      "it": 275,
+      "an": 276,
+      "ar": 277,
+      "al": 278,
+      "Ġthe": 279,
+      ";Ċ": 280,
+      "Ġp": 281,
+      "Ġf": 282,
+      "ou": 283,
+      "Ġ=": 284,
+      "is": 285,
+      "ĠĠĠĠĠĠĠ": 286,
+      "ing": 287,
+      "es": 288,
+      "Ġw": 289,
+      "ion": 290,
+      "ed": 291,
+      "ic": 292,
+      "Ġb": 293,
+      "Ġd": 294,
+      "et": 295,
+      "Ġm": 296,
+      "Ġo": 297,
+      "ĉĉ": 298,
+      "ro": 299,
+      "as": 300,
+      "el": 301,
+      "ct": 302,
+      "nd": 303,
+      "Ġin": 304,
+      "Ġh": 305,
+      "ent": 306,
+      "id": 307,
+      "Ġn": 308,
+      "am": 309,
+      "ĠĠĠĠĠĠĠĠĠĠĠ": 310,
+      "Ġto": 311,
+      "Ġre": 312,
+      "--": 313,
+      "Ġ{": 314,
+      "Ġof": 315,
+      "om": 316,
+      ");Ċ": 317,
+      "im": 318,
+      "čĊ": 319,
+      "Ġ(": 320,
+      "il": 321,
+      "//": 322,
+      "Ġand": 323,
+      "ur": 324,
+      "se": 325,
+      "Ġl": 326,
+      "ex": 327,
+      "ĠS": 328,
+      "ad": 329,
+      "Ġ\"": 330,
+      "ch": 331,
+      "ut": 332,
+      "if": 333,
+      "**": 334,
+      "Ġ}": 335,
+      "em": 336,
+      "ol": 337,
+      "ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ": 338,
+      "th": 339,
+      ")Ċ": 340,
+      "Ġ{Ċ": 341,
+      "Ġg": 342,
+      "ig": 343,
+      "iv": 344,
+      ",Ċ": 345,
+      "ce": 346,
+      "od": 347,
+      "Ġv": 348,
+      "ate": 349,
+      "ĠT": 350,
+      "ag": 351,
+      "ay": 352,
+      "Ġ*": 353,
+      "ot": 354,
+      "us": 355,
+      "ĠC": 356,
+      "Ġst": 357,
+      "ĠI": 358,
+      "un": 359,
+      "ul": 360,
+      "ue": 361,
+      "ĠA": 362,
+      "ow": 363,
+      "Ġ'": 364,
+      "ew": 365,
+      "Ġ<": 366,
+      "ation": 367,
+      "()": 368,
+      "Ġfor": 369,
+      "ab": 370,
+      "ort": 371,
+      "um": 372,
+      "ame": 373,
+      "Ġis": 374,
+      "pe": 375,
+      "tr": 376,
+      "ck": 377,
+      "âĢ": 378,
+      "Ġy": 379,
+      "ist": 380,
+      "----": 381,
+      ".ĊĊ": 382,
+      "he": 383,
+      "Ġe": 384,
+      "lo": 385,
+      "ĠM": 386,
+      "Ġbe": 387,
+      "ers": 388,
+      "Ġon": 389,
+      "Ġcon": 390,
+      "ap": 391,
+      "ub": 392,
+      "ĠP": 393,
+      "ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ": 394,
+      "ass": 395,
+      "int": 396,
+      ">Ċ": 397,
+      "ly": 398,
+      "urn": 399,
+      "Ġ$": 400,
+      ";ĊĊ": 401,
+      "av": 402,
+      "port": 403,
+      "ir": 404,
+      "->": 405,
+      "nt": 406,
+      "ction": 407,
+      "end": 408,
+      "Ġde": 409,
+      "ith": 410,
+      "out": 411,
+      "turn": 412,
+      "our": 413,
+      "ĠĠĠĠĠ": 414,
+      "lic": 415,
+      "res": 416,
+      "pt": 417,
+      "==": 418,
+      "Ġthis": 419,
+      "Ġwh": 420,
+      "Ġif": 421,
+      "ĠD": 422,
+      "ver": 423,
+      "age": 424,
+      "ĠB": 425,
+      "ht": 426,
+      "ext": 427,
+      "=\"": 428,
+      "Ġthat": 429,
+      "****": 430,
+      "ĠR": 431,
+      "Ġit": 432,
+      "ess": 433,
+      "ĠF": 434,
+      "Ġr": 435,
+      "os": 436,
+      "and": 437,
+      "Ġas": 438,
+      "ect": 439,
+      "ke": 440,
+      "rom": 441,
+      "Ġ//": 442,
+      "con": 443,
+      "ĠL": 444,
+      "(\"": 445,
+      "qu": 446,
+      "lass": 447,
+      "Ġwith": 448,
+      "iz": 449,
+      "de": 450,
+      "ĠN": 451,
+      "Ġal": 452,
+      "op": 453,
+      "up": 454,
+      "get": 455,
+      "Ġ}Ċ": 456,
+      "ile": 457,
+      "Ġan": 458,
+      "ata": 459,
+      "ore": 460,
+      "ri": 461,
+      "Ġpro": 462,
+      ";čĊ": 463,
+      "ĉĉĉĉ": 464,
+      "ter": 465,
+      "ain": 466,
+      "ĠW": 467,
+      "ĠE": 468,
+      "Ġcom": 469,
+      "Ġreturn": 470,
+      "art": 471,
+      "ĠH": 472,
+      "ack": 473,
+      "import": 474,
+      "ublic": 475,
+      "Ġor": 476,
+      "est": 477,
+      "ment": 478,
+      "ĠG": 479,
+      "able": 480,
+      "Ġ-": 481,
+      "ine": 482,
+      "ill": 483,
+      "ind": 484,
+      "ere": 485,
+      "::": 486,
+      "ity": 487,
+      "Ġ+": 488,
+      "Ġtr": 489,
+      "elf": 490,
+      "ight": 491,
+      "('": 492,
+      "orm": 493,
+      "ult": 494,
+      "str": 495,
+      "..": 496,
+      "\",": 497,
+      "Ġyou": 498,
+      "ype": 499
+    },
+    "merges": [
+      "Ġ Ġ",
+      "ĠĠ ĠĠ",
+      "i n",
+      "Ġ t",
+      "ĠĠĠĠ ĠĠĠĠ",
+      "e r",
+      "ĠĠ Ġ",
+      "o n",
+      "Ġ a",
+      "r e",
+      "a t",
+      "s t",
+      "e n",
+      "o r",
+      "Ġt h",
+      "Ċ Ċ",
+      "Ġ c",
+      "l e",
+      "Ġ s",
+      "i t",
+      "a n",
+      "a r",
+      "a l",
+      "Ġth e",
+      "; Ċ",
+      "Ġ p",
+      "Ġ f",
+      "o u",
+      "Ġ =",
+      "i s",
+      "ĠĠĠĠ ĠĠĠ",
+      "in g",
+      "e s",
+      "Ġ w",
+      "i on",
+      "e d",
+      "i c",
+      "Ġ b",
+      "Ġ d",
+      "e t",
+      "Ġ m",
+      "Ġ o",
+      "ĉ ĉ",
+      "r o",
+      "a s",
+      "e l",
+      "c t",
+      "n d",
+      "Ġ in",
+      "Ġ h",
+      "en t",
+      "i d",
+      "Ġ n",
+      "a m",
+      "ĠĠĠĠĠĠĠĠ ĠĠĠ",
+      "Ġt o",
+      "Ġ re",
+      "- -",
+      "Ġ {",
+      "Ġo f",
+      "o m",
+      ") ;Ċ",
+      "i m",
+      "č Ċ",
+      "Ġ (",
+      "i l",
+      "/ /",
+      "Ġa nd",
+      "u r",
+      "s e",
+      "Ġ l",
+      "e x",
+      "Ġ S",
+      "a d",
+      "Ġ \"",
+      "c h",
+      "u t",
+      "i f",
+      "* *",
+      "Ġ }",
+      "e m",
+      "o l",
+      "ĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠ",
+      "t h",
+      ") Ċ",
+      "Ġ{ Ċ",
+      "Ġ g",
+      "i g",
+      "i v",
+      ", Ċ",
+      "c e",
+      "o d",
+      "Ġ v",
+      "at e",
+      "Ġ T",
+      "a g",
+      "a y",
+      "Ġ *",
+      "o t",
+      "u s",
+      "Ġ C",
+      "Ġ st",
+      "Ġ I",
+      "u n",
+      "u l",
+      "u e",
+      "Ġ A",
+      "o w",
+      "Ġ '",
+      "e w",
+      "Ġ <",
+      "at ion",
+      "( )",
+      "Ġf or",
+      "a b",
+      "or t",
+      "u m",
+      "am e",
+      "Ġ is",
+      "p e",
+      "t r",
+      "c k",
+      "â Ģ",
+      "Ġ y",
+      "i st",
+      "-- --",
+      ". ĊĊ",
+      "h e",
+      "Ġ e",
+      "l o",
+      "Ġ M",
+      "Ġb e",
+      "er s",
+      "Ġ on",
+      "Ġc on",
+      "a p",
+      "u b",
+      "Ġ P",
+      "ĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠ",
+      "as s",
+      "in t",
+      "> Ċ",
+      "l y",
+      "ur n",
+      "Ġ $",
+      "; ĊĊ",
+      "a v",
+      "p ort",
+      "i r",
+      "- >",
+      "n t",
+      "ct ion",
+      "en d",
+      "Ġd e",
+      "it h",
+      "ou t",
+      "t urn",
+      "ou r",
+      "ĠĠĠĠ Ġ",
+      "l ic",
+      "re s",
+      "p t",
+      "= =",
+      "Ġth is",
+      "Ġw h",
+      "Ġ if",
+      "Ġ D",
+      "v er",
+      "ag e",
+      "Ġ B",
+      "h t",
+      "ex t",
+      "= \"",
+      "Ġth at",
+      "** **",
+      "Ġ R",
+      "Ġ it",
+      "es s",
+      "Ġ F",
+      "Ġ r",
+      "o s",
+      "an d",
+      "Ġa s",
+      "e ct",
+      "k e",
+      "ro m",
+      "Ġ //",
+      "c on",
+      "Ġ L",
+      "( \"",
+      "q u",
+      "l ass",
+      "Ġw ith",
+      "i z",
+      "d e",
+      "Ġ N",
+      "Ġa l",
+      "o p",
+      "u p",
+      "g et",
+      "Ġ} Ċ",
+      "i le",
+      "Ġa n",
+      "at a",
+      "o re",
+      "r i",
+      "Ġp ro",
+      "; čĊ",
+      "ĉĉ ĉĉ",
+      "t er",
+      "a in",
+      "Ġ W",
+      "Ġ E",
+      "Ġc om",
+      "Ġre turn",
+      "ar t",
+      "Ġ H",
+      "a ck",
+      "im port",
+      "ub lic",
+      "Ġ or",
+      "e st",
+      "m ent",
+      "Ġ G",
+      "ab le",
+      "Ġ -",
+      "in e",
+      "il l",
+      "in d",
+      "er e",
+      ": :",
+      "it y",
+      "Ġ +",
+      "Ġt r",
+      "el f",
+      "ig ht",
+      "( '",
+      "or m",
+      "ul t",
+      "st r",
+      ". .",
+      "\" ,",
+      "Ġy ou",
+      "y pe"
+    ]
+  }
+}

tokenizer_config.json

@@ -0,0 +1,143 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "chat_template": "{% set image_count = namespace(value=0) %}{% set video_count = namespace(value=0) %}{% for message in messages %}{% if loop.first and message['role'] != 'system' %}<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n{% endif %}<|im_start|>{{ message['role'] }}\n{% if message['content'] is string %}{{ message['content'] }}<|im_end|>\n{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}{% set image_count.value = image_count.value + 1 %}{% if add_vision_id %}Picture {{ image_count.value }}: {% endif %}<|vision_start|><|image_pad|><|vision_end|>{% elif content['type'] == 'video' or 'video' in content %}{% set video_count.value = video_count.value + 1 %}{% if add_vision_id %}Video {{ video_count.value }}: {% endif %}<|vision_start|><|video_pad|><|vision_end|>{% elif 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}<|im_end|>\n{% endif %}{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant\n{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "padding_side": "left",
+  "errors": "replace",
+  "model_max_length": 32768,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}