Upload folder using huggingface_hub

added_tokens.json

@@ -1,17 +1,38 @@
 {
-  "</box>": 151654,
-  "</img>": 151647,
-  "</p>": 151657,
-  "</quad>": 151650,
-  "</ref>": 151652,
-  "<IMG_CONTEXT>": 151648,
-  "<box>": 151653,
-  "<img>": 151646,
-  "<p>": 151656,
-  "<quad>": 151649,
-  "<ref>": 151651,
+  "</box>": 151673,
+  "</img>": 151666,
+  "</p>": 151676,
+  "</quad>": 151669,
+  "</ref>": 151671,
+  "</tool_call>": 151658,
+  "</vp>": 151678,
+  "<IMG_CONTEXT>": 151667,
+  "<box>": 151672,
+  "<img>": 151665,
+  "<p>": 151675,
+  "<quad>": 151668,
+  "<ref>": 151670,
+  "<tool_call>": 151657,
+  "<vp>": 151677,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
   "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
   "<|im_end|>": 151645,
   "<|im_start|>": 151644,
-  "[SEG]": 151655
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652,
+  "[SEG]": 151674
 }

config.json

@@ -13,7 +13,7 @@
   "force_image_size": 448,
   "hidden_size": 896,
   "llm_config": {
-    "_name_or_path": "Qwen/Qwen2-0.5B-Instruct",
+    "_name_or_path": "Qwen/Qwen2.5-0.5B-Instruct",
     "add_cross_attention": false,
     "architectures": [
       "Qwen2ForCausalLM"
@@ -51,7 +51,7 @@
     "length_penalty": 1.0,
     "max_length": 20,
     "max_position_embeddings": 32768,
-    "max_window_layers": 24,
+    "max_window_layers": 21,
     "min_length": 0,
     "model_type": "qwen2",
     "no_repeat_ngram_size": 0,
@@ -81,7 +81,7 @@
     "temperature": 1.0,
     "tf_legacy_loss": false,
     "tie_encoder_decoder": false,
-    "tie_word_embeddings": true,
+    "tie_word_embeddings": false,
     "tokenizer_class": null,
     "top_k": 50,
     "top_p": 1.0,
@@ -92,7 +92,7 @@
     "use_bfloat16": true,
     "use_cache": true,
     "use_sliding_window": false,
-    "vocab_size": 151658
+    "vocab_size": 151679
   },
   "max_dynamic_patch": 12,
   "min_dynamic_patch": 1,

special_tokens_map.json

@@ -2,6 +2,17 @@
   "additional_special_tokens": [
     "<|im_start|>",
     "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>",
     "<img>",
     "</img>",
     "<IMG_CONTEXT>",

tokenizer.json

@@ -32,7 +32,7 @@
     },
     {
       "id": 151646,
-      "content": "<img>",
+      "content": "<|object_ref_start|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -41,7 +41,7 @@
     },
     {
       "id": 151647,
-      "content": "</img>",
+      "content": "<|object_ref_end|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -50,7 +50,7 @@
     },
     {
       "id": 151648,
-      "content": "<IMG_CONTEXT>",
+      "content": "<|box_start|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -59,7 +59,7 @@
     },
     {
       "id": 151649,
-      "content": "<quad>",
+      "content": "<|box_end|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -68,7 +68,7 @@
     },
     {
       "id": 151650,
-      "content": "</quad>",
+      "content": "<|quad_start|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -77,7 +77,7 @@
     },
     {
       "id": 151651,
-      "content": "<ref>",
+      "content": "<|quad_end|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -86,7 +86,7 @@
     },
     {
       "id": 151652,
-      "content": "</ref>",
+      "content": "<|vision_start|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -95,7 +95,7 @@
     },
     {
       "id": 151653,
-      "content": "<box>",
+      "content": "<|vision_end|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -104,7 +104,7 @@
     },
     {
       "id": 151654,
-      "content": "</box>",
+      "content": "<|vision_pad|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -113,7 +113,7 @@
     },
     {
       "id": 151655,
-      "content": "[SEG]",
+      "content": "<|image_pad|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -122,7 +122,7 @@
     },
     {
       "id": 151656,
-      "content": "<p>",
+      "content": "<|video_pad|>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -131,12 +131,201 @@
     },
     {
       "id": 151657,
+      "content": "<tool_call>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": false
+    },
+    {
+      "id": 151658,
+      "content": "</tool_call>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": false
+    },
+    {
+      "id": 151659,
+      "content": "<|fim_prefix|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": false
+    },
+    {
+      "id": 151660,
+      "content": "<|fim_middle|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": false
+    },
+    {
+      "id": 151661,
+      "content": "<|fim_suffix|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": false
+    },
+    {
+      "id": 151662,
+      "content": "<|fim_pad|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": false
+    },
+    {
+      "id": 151663,
+      "content": "<|repo_name|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": false
+    },
+    {
+      "id": 151664,
+      "content": "<|file_sep|>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": false
+    },
+    {
+      "id": 151665,
+      "content": "<img>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 151666,
+      "content": "</img>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 151667,
+      "content": "<IMG_CONTEXT>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 151668,
+      "content": "<quad>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 151669,
+      "content": "</quad>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 151670,
+      "content": "<ref>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 151671,
+      "content": "</ref>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 151672,
+      "content": "<box>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 151673,
+      "content": "</box>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 151674,
+      "content": "[SEG]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 151675,
+      "content": "<p>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 151676,
       "content": "</p>",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
       "normalized": false,
       "special": true
+    },
+    {
+      "id": 151677,
+      "content": "<vp>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 151678,
+      "content": "</vp>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
     }
   ],
   "normalizer": {
@@ -152137,6 +152326,7 @@
       "o k",
       "_ _",
       "c l",
+      "# #",
       "Ġh e",
       "ar d",
       ") .",
@@ -152391,6 +152581,7 @@
       "Ġp er",
       "C ont",
       "ing s",
+      "## ##",
       "Ġd ata",
       "Ġs a",
       "e f",
@@ -152637,6 +152828,7 @@
       "is e",
       "m in",
       "ĠĠĠĠ Ċ",
+      "# include",
       "eth od",
       ". P",
       "ut e",
@@ -152926,6 +153118,7 @@
       "Ex ception",
       "in put",
       "Ġtr ans",
+      "#### ####",
       "ord er",
       "B y",
       "Ġa w",
@@ -153009,6 +153202,7 @@
       "Ġb reak",
       "Ġ );Ċ",
       "re n",
+      "# define",
       "it t",
       "Ġa p",
       "ĉ c",
@@ -153998,6 +154192,7 @@
       "http s",
       "Ġd em",
       "Ġe ar",
+      "######## ########",
       "Ġm atch",
       "or ies",
       "ac es",
@@ -154219,6 +154414,7 @@
       "ER R",
       "Ġ< =",
       "at ely",
+      "# if",
       "u ction",
       "ĠT e",
       "Ġl ink",
@@ -154228,6 +154424,7 @@
       "m e",
       "Ġg iven",
       "O ut",
+      "# endif",
       "Ġbet ter",
       "P age",
       "Ġfe el",
@@ -156520,6 +156717,7 @@
       "o ch",
       "_ image",
       "\\ t",
+      "# Ċ",
       "( L",
       "Ġindu stry",
       "com ing",
@@ -156589,6 +156787,7 @@
       "Ġcurrent ly",
       "se c",
       "Ġrelations hip",
+      "################ ################",
       "ĠM ap",
       "as et",
       "Ġparam eters",
@@ -157090,6 +157289,7 @@
       "ast e",
       "Ġpro file",
       "Ġread y",
+      "#if ndef",
       "ro te",
       "Ġs ense",
       "G ener",
@@ -157314,6 +157514,7 @@
       "Ġp en",
       ".b tn",
       "ĠA S",
+      "#if def",
       "Ġcho ice",
       "ĠP age",
       "_P RO",
@@ -157473,6 +157674,7 @@
       "S cript",
       "d at",
       "Ġr ule",
+      "# import",
       "=\" /",
       "S erial",
       "Ġstart ing",
@@ -158049,6 +158251,7 @@
       "ib ly",
       ".e quals",
       "Ġintern ational",
+      "# pragma",
       "oo th",
       "W riter",
       "i ate",
@@ -159568,6 +159771,7 @@
       "ĠH ot",
       "ĠInd ex",
       "; &",
+      "# !",
       "ĠN or",
       "ĠC ap",
       "- (",
@@ -159722,6 +159926,7 @@
       "ĠApp le",
       "g in",
       "Re quired",
+      "# +",
       "land s",
       "Ġs qu",
       "Ġfact or",
@@ -160257,6 +160462,7 @@
       "i ro",
       "Ġrem ote",
       "ĠI T",
+      "#! /",
       "Ġred istrib",
       "ra z",
       "ĠS ince",
@@ -161494,6 +161700,7 @@
       "ĠJ ournal",
       "r outer",
       "Ġmys qli",
+      "# else",
       ") \"",
       "-x s",
       "let s",
@@ -163582,6 +163789,7 @@
       "-d anger",
       "Ġroom s",
       "con v",
+      "# {",
       ". op",
       "ĠA rea",
       "_S C",
@@ -164620,6 +164828,7 @@
       "Ġfact s",
       "Ġun t",
       ".in stance",
+      "################################ ################################",
       "- end",
       "ĠJO IN",
       "ĠH en",
@@ -164905,6 +165114,7 @@
       "ĠJew ish",
       "Ġre covery",
       "Ġstand s",
+      "# [",
       "Ġafter noon",
       "ĠArt icle",
       "_ att",
@@ -165440,6 +165650,7 @@
       "err a",
       "Ċ ĠĠĊ",
       "utor ial",
+      "# a",
       "p ay",
       "char ge",
       "ĠI re",
@@ -165924,6 +166135,7 @@
       "Ġstre ets",
       "_M SG",
       "New s",
+      "## #",
       ": /",
       "Ġcut ting",
       "x B",
@@ -167739,6 +167951,7 @@
       "gener ated",
       "Ġad mitted",
       "Ġp ussy",
+      "# w",
       "Ġsw im",
       "un ion",
       "N a",
@@ -167790,6 +168003,7 @@
       "vel y",
       "} );čĊ",
       "_ ENT",
+      "#+ #+",
       "art icles",
       "��Sou thern",
       "Ġstrong er",
@@ -169002,6 +169216,7 @@
       "Ġcharacter istics",
       "D one",
       "el n",
+      "######## ####",
       "PO S",
       "Ġd ensity",
       "ĠPl atform",
@@ -170006,6 +170221,7 @@
       "d og",
       "Ġclick ing",
       "), ĊĊ",
+      "# line",
       "Oper ator",
       "Ġc iv",
       "Ġm erg",
@@ -171417,6 +171633,7 @@
       "Ġdemonstr ate",
       "ĠHow ard",
       "D rop",
+      "# undef",
       "Ġinv oke",
       "ĠB ridge",
       "end en",
@@ -174158,6 +174375,7 @@
       "ĠP ub",
       "Ġâ Ķ",
       "c ion",
+      "## Ċ",
       "II I",
       "Tag Name",
       "Ġam id",
@@ -175207,6 +175425,7 @@
       ".m aterial",
       "ĠD ue",
       "ĠP el",
+      "# print",
       "Ġindepend ence",
       "ex us",
       "Sh adow",
@@ -178029,6 +178248,7 @@
       "ĠFant asy",
       "st ory",
       "Ġm ême",
+      "# ĊĊ",
       "_s lice",
       "olt age",
       "H ar",
@@ -178381,6 +178601,7 @@
       "am ination",
       "[ @\"",
       "Ġm uj",
+      "## #Ċ",
       "First OrDefault",
       "then Return",
       "C he",
@@ -178552,6 +178773,7 @@
       "ather ine",
       "( dest",
       "az ed",
+      "# endregion",
       "sem bl",
       ", M",
       "ob y",
@@ -180388,6 +180610,7 @@
       "ib o",
       "Ġlo yal",
       "Ġuse less",
+      "# elif",
       "ĠUlt imate",
       "C ome",
       "g el",
@@ -180595,6 +180818,7 @@
       "ation ally",
       "ĠMe eting",
       "è¯ ¯",
+      "# region",
       "Ġrout ing",
       ".f ocus",
       "ĠYou th",
@@ -181391,6 +181615,7 @@
       ".prot ocol",
       "AF E",
       "Ġtext ures",
+      "################################ ################",
       "umb ai",
       ".st ats",
       "ĠG E",
@@ -183577,6 +183802,7 @@
       ".F ull",
       ". undefined",
       "ĠSequ elize",
+      "################################################################ ############",
       "Ġeduc ated",
       "_O VER",
       "åº ı",
@@ -185188,6 +185414,7 @@
       "dd s",
       "Ġdepos its",
       "ĉd river",
+      "# ga",
       "pr ising",
       "print ln",
       "Ġpres enter",
@@ -192020,6 +192247,7 @@
       "ac ific",
       ".v olume",
       "Ġmir rors",
+      "################################################################ ################",
       "Ġviol ate",
       "ars ers",
       "Ġsoc io",
@@ -192877,6 +193105,7 @@
       "ar de",
       "Ġfier ce",
       "lic ted",
+      "# čĊ",
       "Ġbreak through",
       "ĠE rik",
       "Ġb ride",
@@ -195619,6 +195848,7 @@
       "rown ed",
       "_m ed",
       "ĉ date",
+      "# __",
       "Ġcost umes",
       "ĠRe quires",
       "aff le",
@@ -197683,6 +197913,7 @@
       "Ġmock ed",
       "ĠT ory",
       "Ġ\") \";Ċ",
+      "# g",
       "Ġli ed",
       "Ġs vc",
       "_g ui",
@@ -199607,10 +199838,12 @@
       "d aily",
       "ĠC oding",
       "( destination",
+      "# $",
       "uj Äħ",
       "Ġemerg ence",
       "_p ara",
       "_IN CLUDE",
+      "# :",
       "Ġrecogn izing",
       "Ġf ug",
       "\"} },Ċ",
@@ -200026,6 +200259,7 @@
       "k p",
       "IT ES",
       "Ġ ################################################################",
+      "# get",
       "/ Desktop",
       "ĉgl m",
       "Ġz inc",
@@ -202051,6 +202285,7 @@
       "Ġdee pest",
       "w k",
       "ĠNo ise",
+      "### ĊĊ",
       "Ġpr éc",
       "ot le",
       "ÑĤ е",
@@ -203850,6 +204085,7 @@
       "-effect ive",
       "Ġsk u",
       "ed ly",
+      "## ĊĊ",
       "ĠH olly",
       "ĠFL ASH",
       "/ TR",
@@ -207068,6 +207304,7 @@
       "_s b",
       "om ens",
       "ĠExec utes",
+      "# \"",
       "TT Y",
       "ĠValue Type",
       "); */Ċ",
@@ -207467,6 +207704,7 @@
       "E OS",
       "H al",
       "Ġtrust worthy",
+      "# =",
       ".EX TRA",
       "Ġman o",
       "is icing",
@@ -212301,6 +212539,7 @@
       "/ place",
       "Ġhol istic",
       ": t",
+      "# ,",
       "Ġb oto",
       "Ġnause a",
       "ĠSh ooting",
@@ -212437,6 +212676,7 @@
       "C ause",
       "at ypes",
       "man ufacturer",
+      "# ----------------------------------------------------------------------------",
       "Ġsp or",
       "es on",
       "Ġpun ched",
@@ -214046,6 +214286,7 @@
       "S creenshot",
       "esthes ia",
       "Ġwalk er",
+      "# from",
       "co ordinate",
       "_ interest",
       "Ġhelp less",
@@ -215256,6 +215497,7 @@
       "Field Type",
       "ok able",
       "ĠRT L",
+      "# index",
       "Ġ% {",
       "Ġar ist",
       ".Get Mapping",
@@ -217578,6 +217820,7 @@
       "so on",
       "Ġdis fr",
       "ĉ Vec",
+      "## _",
       ".s chool",
       "Ġbl inds",
       "Ġac ab",
@@ -218849,6 +219092,7 @@
       "ĉex ec",
       "EN AME",
       "_ letters",
+      "#### #",
       "ĠC s",
       "'] ==\"",
       "Ġ\" ')",
@@ -219050,6 +219294,7 @@
       "Ġtem pl",
       "ĠExp ense",
       "e ight",
+      "# SBATCH",
       "z ones",
       ".p arts",
       "at rice",
@@ -219384,6 +219629,7 @@
       "i ard",
       "Theme Provider",
       "Ġevent Data",
+      "# ad",
       ".get Url",
       "Ġtool box",
       "Ġover riding",
@@ -220228,6 +220474,7 @@
       "ĠTh eta",
       "_inter p",
       "R aster",
+      "# error",
       ", obj",
       "Ġtweet ing",
       "_G PU",
@@ -220575,6 +220822,7 @@
       "Ġè¾ĵ åħ¥",
       "Ġintimid ation",
       "end ale",
+      "################################################################ ########",
       "Ġinsight ful",
       "Ġs ands",
       "Ġphotograph ic",
@@ -220980,6 +221228,7 @@
       "Dep loy",
       ".Con tract",
       "- bo",
+      "# /",
       "Ġinter ception",
       "Ġis bn",
       "Ġman ners",
@@ -221462,6 +221711,7 @@
       "\\ \\\"",
       "- job",
       "Ġsevent y",
+      "#### Ċ",
       "ĠMan or",
       "Ġdown right",
       "Ġtime frame",
@@ -223267,6 +223517,7 @@
       "/m essages",
       "Ġof Type",
       "ĉs wap",
+      "# plt",
       "ĠTur ks",
       "N ES",
       "Ġprogress ively",
@@ -223514,6 +223765,7 @@
       "_T itle",
       "(G tk",
       "Ġc elle",
+      "# ================================================================",
       "ĠJ oomla",
       "\"> //",
       "Month ly",
@@ -227019,6 +227271,7 @@
       "ĠRevel ation",
       "Ġrap ide",
       "p unk",
+      "# ----------------------------------------------------------------",
       "Object Id",
       "abin et",
       "extr acomment",
@@ -227423,6 +227676,7 @@
       "(c d",
       "Ġun zip",
       "Ġglam orous",
+      "# \",",
       "Ġn aw",
       "Ġmin ib",
       "ĠBr an",
@@ -227705,6 +227959,7 @@
       "Ġragaz za",
       "/ tag",
       "Ġirres ist",
+      "# End",
       "****** *čĊ",
       "Ġrestr ained",
       "Ġch iropr",
@@ -227842,6 +228097,7 @@
       "Ġgr âce",
       "ĠH elsinki",
       "G ro",
+      "# af",
       "ìĭ Ŀ",
       "Ġsou ha",
       "ĠInd ie",
@@ -228634,6 +228890,7 @@
       ".M ESSAGE",
       "(de g",
       "å¿ Ĺ",
+      "#### ##",
       "Ġ\"\" ),",
       "kl är",
       "\\M ail",
@@ -228848,6 +229105,7 @@
       "/ grid",
       "Ġfil thy",
       ".e v",
+      "#### #Ċ",
       "Ġs are",
       "Ġso aking",
       "ĠReg ions",
@@ -229088,6 +229346,7 @@
       "` ;",
       "ãĥ ĭ",
       "cent e",
+      "# ab",
       "Ġlex ical",
       "ĠB RO",
       "Ġr ulings",
@@ -229196,6 +229455,7 @@
       "ĉ props",
       "Ġrot ten",
       "Re jected",
+      "# ac",
       ". ua",
       "ĠAm nesty",
       "Ġpenn ed",
@@ -229538,6 +229798,7 @@
       "cul os",
       "_person al",
       "Ġanaly tic",
+      "# create",
       "_mem cpy",
       "(List Node",
       "_T ag",
@@ -230455,6 +230716,7 @@
       "ìŀħ ëĭĪëĭ¤",
       "Ġsid eline",
       ".Array Adapter",
+      "#### ##Ċ",
       "ĠSy rians",
       "ĠAtt endance",
       "-es que",
@@ -230947,6 +231209,7 @@
       "ALL ERY",
       "c j",
       "x AD",
+      "############################################################################ ###Ċ",
       "Ġitalian i",
       "| #",
       "Ġreg enerate",
@@ -231915,6 +232178,7 @@
       "_C ategory",
       "ĠBT N",
       "ĠDar th",
+      "# for",
       "eth nic",
       "arch itecture",
       "ĠCou pe",
@@ -232989,6 +233253,7 @@
       "Ġcooper ating",
       "ung i",
       "Ar izona",
+      "# echo",
       "-ex pression",
       ".min utes",
       "Ġpref ixed",
@@ -233520,6 +233785,7 @@
       "ĠMom ents",
       "enu ous",
       "IFIC ATE",
+      "#### ###Ċ",
       "æĸĩ ç«ł",
       "á»į c",
       "orm sg",
@@ -233567,6 +233833,7 @@
       "ĠIS IL",
       "Ġco op",
       "h ores",
+      "# @",
       "Ġcomp el",
       "(s kip",
       "éĺ ³",
@@ -233791,6 +234058,7 @@
       "Ġ-*- čĊ",
       "if es",
       "ĠM ansion",
+      "# Region",
       "C ancellation",
       "Ġnear ing",
       "Ġl angu",
@@ -234071,6 +234339,7 @@
       "Ġw reak",
       "Mar vel",
       "/s l",
+      "################################ ########",
       "Ġmov able",
       "Ñĥ й",
       "ĠDr inking",
@@ -235646,6 +235915,7 @@
       "ĠPal o",
       "ĠSuz anne",
       "_m appings",
+      "#{ @",
       "ĠOccup ational",
       "_b uckets",
       "go als",
@@ -235790,6 +236060,7 @@
       "-w atch",
       "ĠHosp itals",
       "} while",
+      "################ ########",
       "á» £",
       "Ġakt ual",
       "Ġkil ograms",
@@ -235911,6 +236182,7 @@
       "Ġcurrent Position",
       "ĠCaucas ian",
       "$ img",
+      "# aa",
       "Ġse an",
       "M ess",
       "*= *=",
@@ -236855,6 +237127,7 @@
       "ĠR udd",
       "ĠPres byterian",
       "und ler",
+      "#! [",
       "_l ahir",
       "() ==\"",
       "Access ibility",
@@ -239334,6 +239607,7 @@
       "_AT OMIC",
       "Sub Menu",
       "_com press",
+      "# .",
       "Dr v",
       ".push Button",
       "Ġsuit case",
@@ -239392,6 +239666,7 @@
       "Ġe books",
       ") size",
       "Ġspecial ised",
+      "# ae",
       "Ġmich ael",
       "ĠSTD OUT",
       "ĠP ell",
@@ -239570,6 +239845,7 @@
       "ĠEisen hower",
       "File d",
       "Ġhe bt",
+      "# \\",
       "create QueryBuilder",
       "æľī æķĪ",
       "v anced",
@@ -239585,6 +239861,7 @@
       "ol vable",
       "Ġal as",
       "(sp ell",
+      "################ ############",
       "Ġdef amation",
       "( Arg",
       "Ġuser Details",
@@ -241522,6 +241799,7 @@
       "Ġsup p",
       "pe on",
       "v rier",
+      "################################################ ########",
       "Ġtrot z",
       "Ġmel tdown",
       "ark ers",
@@ -242213,6 +242491,7 @@
       "ser ter",
       "Ġstore front",
       "_ ends",
+      "################################################################################ Ċ",
       "ĉg it",
       "D SP",
       "CH AIN",
@@ -243735,6 +244014,7 @@
       "ĠST ILL",
       "Qualified Name",
       ". RES",
+      "# c",
       ".w riteln",
       "ĠImmutable List",
       "ĠTh umb",
@@ -244716,6 +244996,7 @@
       "in ne",
       "ĠCONST ANTS",
       "_P rice",
+      "# %%",
       "Ġar sch",
       "ĠNS AttributedString",
       "ĠFile Type",
@@ -245662,6 +245943,7 @@
       "_P ICTURE",
       "_OR IENTATION",
       "ĠO PP",
+      "# '",
       "áf ico",
       ".h istogram",
       "ĠB enny",
@@ -245932,6 +246214,7 @@
       "ew n",
       "Ġpop ulous",
       "ĠSh ed",
+      "# ${",
       "ĠA lo",
       "Device Info",
       "(IN VOKE",
@@ -249322,6 +249605,7 @@
       "ĠD ISTRIBUT",
       "é ments",
       "ĠValid ates",
+      "################################################ ############",
       "Ġ| /",
       "Ġes l",
       "Ġré seau",
@@ -249341,6 +249625,7 @@
       "ime i",
       ".get Bounds",
       ".Mouse EventHandler",
+      "#### ###",
       "Generic Type",
       "/c ms",
       "Ġturn o",

tokenizer_config.json

@@ -1,4 +1,5 @@
 {
+  "add_bos_token": false,
   "add_eos_token": false,
   "add_prefix_space": false,
   "added_tokens_decoder": {
@@ -27,7 +28,7 @@
       "special": true
     },
     "151646": {
-      "content": "<img>",
+      "content": "<|object_ref_start|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -35,7 +36,7 @@
       "special": true
     },
     "151647": {
-      "content": "</img>",
+      "content": "<|object_ref_end|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -43,7 +44,7 @@
       "special": true
     },
     "151648": {
-      "content": "<IMG_CONTEXT>",
+      "content": "<|box_start|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -51,7 +52,7 @@
       "special": true
     },
     "151649": {
-      "content": "<quad>",
+      "content": "<|box_end|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -59,7 +60,7 @@
       "special": true
     },
     "151650": {
-      "content": "</quad>",
+      "content": "<|quad_start|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -67,7 +68,7 @@
       "special": true
     },
     "151651": {
-      "content": "<ref>",
+      "content": "<|quad_end|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -75,7 +76,7 @@
       "special": true
     },
     "151652": {
-      "content": "</ref>",
+      "content": "<|vision_start|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -83,7 +84,7 @@
       "special": true
     },
     "151653": {
-      "content": "<box>",
+      "content": "<|vision_end|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -91,7 +92,7 @@
       "special": true
     },
     "151654": {
-      "content": "</box>",
+      "content": "<|vision_pad|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -99,7 +100,7 @@
       "special": true
     },
     "151655": {
-      "content": "[SEG]",
+      "content": "<|image_pad|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -107,7 +108,7 @@
       "special": true
     },
     "151656": {
-      "content": "<p>",
+      "content": "<|video_pad|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -115,17 +116,196 @@
       "special": true
     },
     "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151665": {
+      "content": "<img>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151666": {
+      "content": "</img>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151667": {
+      "content": "<IMG_CONTEXT>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151668": {
+      "content": "<quad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151669": {
+      "content": "</quad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151670": {
+      "content": "<ref>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151671": {
+      "content": "</ref>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151672": {
+      "content": "<box>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151673": {
+      "content": "</box>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151674": {
+      "content": "[SEG]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151675": {
+      "content": "<p>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151676": {
       "content": "</p>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
+    },
+    "151677": {
+      "content": "<vp>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151678": {
+      "content": "</vp>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
     }
   },
   "additional_special_tokens": [
     "<|im_start|>",
     "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>",
     "<img>",
     "</img>",
     "<IMG_CONTEXT>",
@@ -137,11 +317,11 @@
     "</box>"
   ],
   "bos_token": null,
-  "chat_template": "{% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ '<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n' }}{% endif %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
+  "chat_template": "{%- if tools %}\n    {{- '<|im_start|>system\\n' }}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- messages[0]['content'] }}\n    {%- else %}\n        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}\n    {%- endif %}\n    {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n    {%- for tool in tools %}\n        {{- \"\\n\" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n    {%- else %}\n        {{- '<|im_start|>system\\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\\n' }}\n    {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n    {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\n        {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n    {%- elif message.role == \"assistant\" %}\n        {{- '<|im_start|>' + message.role }}\n        {%- if message.content %}\n            {{- '\\n' + message.content }}\n        {%- endif %}\n        {%- for tool_call in message.tool_calls %}\n            {%- if tool_call.function is defined %}\n                {%- set tool_call = tool_call.function %}\n            {%- endif %}\n            {{- '\\n<tool_call>\\n{\"name\": \"' }}\n            {{- tool_call.name }}\n            {{- '\", \"arguments\": ' }}\n            {{- tool_call.arguments | tojson }}\n            {{- '}\\n</tool_call>' }}\n        {%- endfor %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"tool\" %}\n        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}\n            {{- '<|im_start|>user' }}\n        {%- endif %}\n        {{- '\\n<tool_response>\\n' }}\n        {{- message.content }}\n        {{- '\\n</tool_response>' }}\n        {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n            {{- '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n",
   "clean_up_tokenization_spaces": false,
   "eos_token": "<|im_end|>",
   "errors": "replace",
-  "model_max_length": 8192,
+  "model_max_length": 16384,
   "pad_token": "<|endoftext|>",
   "padding_side": "right",
   "split_special_tokens": false,