tiny-random-molmo2 / tokenizer_config.json
rkazants's picture
Upload tiny random Molmo2 (allenai/MolmoWeb-4B architecture) (#1)
ae23feb
Raw History Blame Contribute Delete
64.4 kB
{ "add_bos_token": false, "add_prefix_space": false, "added_tokens_decoder": { "151643": { "content": "<|endoftext|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151644": { "content": "<|im_start|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151645": { "content": "<|im_end|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151646": { "content": "<|object_ref_start|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151647": { "content": "<|object_ref_end|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151648": { "content": "<|box_start|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151649": { "content": "<|box_end|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151650": { "content": "<|quad_start|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151651": { "content": "<|quad_end|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151652": { "content": "<|vision_start|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151653": { "content": "<|vision_end|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151654": { "content": "<|vision_pad|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151655": { "content": "<|image_pad|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151656": { "content": "<|video_pad|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151657": { "content": "<tool_call>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "151658": { "content": "</tool_call>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "151659": { "content": "<|fim_prefix|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "151660": { "content": "<|fim_middle|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "151661": { "content": "<|fim_suffix|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "151662": { "content": "<|fim_pad|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "151663": { "content": "<|repo_name|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "151664": { "content": "<|file_sep|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "151665": { "content": "<tool_response>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "151666": { "content": "</tool_response>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "151667": { "content": "<think>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "151668": { "content": "</think>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "151669": { "content": "|<EXTRA_TOKENS_0>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151670": { "content": "|<EXTRA_TOKENS_1>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151671": { "content": "|<EXTRA_TOKENS_2>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151672": { "content": "|<EXTRA_TOKENS_3>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151673": { "content": "|<EXTRA_TOKENS_4>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151674": { "content": "|<EXTRA_TOKENS_5>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151675": { "content": "|<EXTRA_TOKENS_6>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151676": { "content": "|<EXTRA_TOKENS_7>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151677": { "content": "|<EXTRA_TOKENS_8>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151678": { "content": "|<EXTRA_TOKENS_9>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151679": { "content": "|<EXTRA_TOKENS_10>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151680": { "content": "|<EXTRA_TOKENS_11>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151681": { "content": "|<EXTRA_TOKENS_12>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151682": { "content": "|<EXTRA_TOKENS_13>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151683": { "content": "|<EXTRA_TOKENS_14>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151684": { "content": "|<EXTRA_TOKENS_15>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151685": { "content": "|<EXTRA_TOKENS_16>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151686": { "content": "|<EXTRA_TOKENS_17>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151687": { "content": "|<EXTRA_TOKENS_18>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151688": { "content": "|<EXTRA_TOKENS_19>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151689": { "content": "|<EXTRA_TOKENS_20>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151690": { "content": "|<EXTRA_TOKENS_21>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151691": { "content": "|<EXTRA_TOKENS_22>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151692": { "content": "|<EXTRA_TOKENS_23>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151693": { "content": "|<EXTRA_TOKENS_24>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151694": { "content": "|<EXTRA_TOKENS_25>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151695": { "content": "|<EXTRA_TOKENS_26>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151696": { "content": "|<EXTRA_TOKENS_27>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151697": { "content": "|<EXTRA_TOKENS_28>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151698": { "content": "|<EXTRA_TOKENS_29>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151699": { "content": "|<EXTRA_TOKENS_30>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151700": { "content": "|<EXTRA_TOKENS_31>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151701": { "content": "|<EXTRA_TOKENS_32>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151702": { "content": "|<EXTRA_TOKENS_33>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151703": { "content": "|<EXTRA_TOKENS_34>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151704": { "content": "|<EXTRA_TOKENS_35>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151705": { "content": "|<EXTRA_TOKENS_36>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151706": { "content": "|<EXTRA_TOKENS_37>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151707": { "content": "|<EXTRA_TOKENS_38>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151708": { "content": "|<EXTRA_TOKENS_39>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151709": { "content": "|<EXTRA_TOKENS_40>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151710": { "content": "|<EXTRA_TOKENS_41>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151711": { "content": "|<EXTRA_TOKENS_42>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151712": { "content": "|<EXTRA_TOKENS_43>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151713": { "content": "|<EXTRA_TOKENS_44>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151714": { "content": "|<EXTRA_TOKENS_45>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151715": { "content": "|<EXTRA_TOKENS_46>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151716": { "content": "|<EXTRA_TOKENS_47>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151717": { "content": "|<EXTRA_TOKENS_48>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151718": { "content": "|<EXTRA_TOKENS_49>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151719": { "content": "|<EXTRA_TOKENS_50>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151720": { "content": "|<EXTRA_TOKENS_51>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151721": { "content": "|<EXTRA_TOKENS_52>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151722": { "content": "|<EXTRA_TOKENS_53>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151723": { "content": "|<EXTRA_TOKENS_54>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151724": { "content": "|<EXTRA_TOKENS_55>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151725": { "content": "|<EXTRA_TOKENS_56>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151726": { "content": "|<EXTRA_TOKENS_57>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151727": { "content": "|<EXTRA_TOKENS_58>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151728": { "content": "|<EXTRA_TOKENS_59>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151729": { "content": "|<EXTRA_TOKENS_60>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151730": { "content": "|<EXTRA_TOKENS_61>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151731": { "content": "|<EXTRA_TOKENS_62>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151732": { "content": "|<EXTRA_TOKENS_63>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151733": { "content": "|<EXTRA_TOKENS_64>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151734": { "content": "|<EXTRA_TOKENS_65>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151735": { "content": "|<EXTRA_TOKENS_66>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151736": { "content": "|<EXTRA_TOKENS_67>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151737": { "content": "|<EXTRA_TOKENS_68>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151738": { "content": "|<EXTRA_TOKENS_69>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151739": { "content": "|<EXTRA_TOKENS_70>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151740": { "content": "|<EXTRA_TOKENS_71>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151741": { "content": "|<EXTRA_TOKENS_72>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151742": { "content": "|<EXTRA_TOKENS_73>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151743": { "content": "|<EXTRA_TOKENS_74>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151744": { "content": "|<EXTRA_TOKENS_75>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151745": { "content": "|<EXTRA_TOKENS_76>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151746": { "content": "|<EXTRA_TOKENS_77>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151747": { "content": "|<EXTRA_TOKENS_78>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151748": { "content": "|<EXTRA_TOKENS_79>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151749": { "content": "|<EXTRA_TOKENS_80>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151750": { "content": "|<EXTRA_TOKENS_81>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151751": { "content": "|<EXTRA_TOKENS_82>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151752": { "content": "|<EXTRA_TOKENS_83>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151753": { "content": "|<EXTRA_TOKENS_84>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151754": { "content": "|<EXTRA_TOKENS_85>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151755": { "content": "|<EXTRA_TOKENS_86>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151756": { "content": "|<EXTRA_TOKENS_87>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151757": { "content": "|<EXTRA_TOKENS_88>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151758": { "content": "|<EXTRA_TOKENS_89>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151759": { "content": "|<EXTRA_TOKENS_90>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151760": { "content": "|<EXTRA_TOKENS_91>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151761": { "content": "|<EXTRA_TOKENS_92>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151762": { "content": "|<EXTRA_TOKENS_93>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151763": { "content": "|<EXTRA_TOKENS_94>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151764": { "content": "|<EXTRA_TOKENS_95>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151765": { "content": "|<EXTRA_TOKENS_96>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151766": { "content": "|<EXTRA_TOKENS_97>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151767": { "content": "|<EXTRA_TOKENS_98>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151768": { "content": "|<EXTRA_TOKENS_99>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151769": { "content": "|<EXTRA_TOKENS_100>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151770": { "content": "|<EXTRA_TOKENS_101>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151771": { "content": "|<EXTRA_TOKENS_102>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151772": { "content": "|<EXTRA_TOKENS_103>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151773": { "content": "|<EXTRA_TOKENS_104>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151774": { "content": "|<EXTRA_TOKENS_105>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151775": { "content": "|<EXTRA_TOKENS_106>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151776": { "content": "|<EXTRA_TOKENS_107>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151777": { "content": "|<EXTRA_TOKENS_108>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151778": { "content": "|<EXTRA_TOKENS_109>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151779": { "content": "|<EXTRA_TOKENS_110>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151780": { "content": "|<EXTRA_TOKENS_111>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151781": { "content": "|<EXTRA_TOKENS_112>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151782": { "content": "|<EXTRA_TOKENS_113>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151783": { "content": "|<EXTRA_TOKENS_114>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151784": { "content": "|<EXTRA_TOKENS_115>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151785": { "content": "|<EXTRA_TOKENS_116>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151786": { "content": "|<EXTRA_TOKENS_117>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151787": { "content": "|<EXTRA_TOKENS_118>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151788": { "content": "|<EXTRA_TOKENS_119>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151789": { "content": "|<EXTRA_TOKENS_120>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151790": { "content": "|<EXTRA_TOKENS_121>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151791": { "content": "|<EXTRA_TOKENS_122>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151792": { "content": "|<EXTRA_TOKENS_123>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151793": { "content": "|<EXTRA_TOKENS_124>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151794": { "content": "|<EXTRA_TOKENS_125>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151795": { "content": "|<EXTRA_TOKENS_126>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151796": { "content": "|<EXTRA_TOKENS_127>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151797": { "content": "|<EXTRA_TOKENS_128>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151798": { "content": "|<EXTRA_TOKENS_129>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151799": { "content": "|<EXTRA_TOKENS_130>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151800": { "content": "|<EXTRA_TOKENS_131>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151801": { "content": "|<EXTRA_TOKENS_132>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151802": { "content": "|<EXTRA_TOKENS_133>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151803": { "content": "|<EXTRA_TOKENS_134>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151804": { "content": "|<EXTRA_TOKENS_135>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151805": { "content": "|<EXTRA_TOKENS_136>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151806": { "content": "|<EXTRA_TOKENS_137>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151807": { "content": "|<EXTRA_TOKENS_138>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151808": { "content": "|<EXTRA_TOKENS_139>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151809": { "content": "|<EXTRA_TOKENS_140>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151810": { "content": "|<EXTRA_TOKENS_141>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151811": { "content": "|<EXTRA_TOKENS_142>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151812": { "content": "|<EXTRA_TOKENS_143>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151813": { "content": "|<EXTRA_TOKENS_144>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151814": { "content": "|<EXTRA_TOKENS_145>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151815": { "content": "|<EXTRA_TOKENS_146>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151816": { "content": "|<EXTRA_TOKENS_147>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151817": { "content": "|<EXTRA_TOKENS_148>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151818": { "content": "|<EXTRA_TOKENS_149>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151819": { "content": "|<EXTRA_TOKENS_150>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151820": { "content": "|<EXTRA_TOKENS_151>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151821": { "content": "|<EXTRA_TOKENS_152>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151822": { "content": "|<EXTRA_TOKENS_153>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151823": { "content": "|<EXTRA_TOKENS_154>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151824": { "content": "|<EXTRA_TOKENS_155>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151825": { "content": "|<EXTRA_TOKENS_156>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151826": { "content": "|<EXTRA_TOKENS_157>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151827": { "content": "|<EXTRA_TOKENS_158>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151828": { "content": "|<EXTRA_TOKENS_159>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151829": { "content": "|<EXTRA_TOKENS_160>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151830": { "content": "|<EXTRA_TOKENS_161>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151831": { "content": "|<EXTRA_TOKENS_162>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151832": { "content": "|<EXTRA_TOKENS_163>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151833": { "content": "|<EXTRA_TOKENS_164>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151834": { "content": "|<EXTRA_TOKENS_165>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151835": { "content": "|<EXTRA_TOKENS_166>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151836": { "content": "|<EXTRA_TOKENS_167>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151837": { "content": "|<EXTRA_TOKENS_168>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151838": { "content": "|<EXTRA_TOKENS_169>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151839": { "content": "|<EXTRA_TOKENS_170>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151840": { "content": "|<EXTRA_TOKENS_171>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151841": { "content": "|<EXTRA_TOKENS_172>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151842": { "content": "|<EXTRA_TOKENS_173>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151843": { "content": "|<EXTRA_TOKENS_174>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151844": { "content": "|<EXTRA_TOKENS_175>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151845": { "content": "|<EXTRA_TOKENS_176>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151846": { "content": "|<EXTRA_TOKENS_177>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151847": { "content": "|<EXTRA_TOKENS_178>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151848": { "content": "|<EXTRA_TOKENS_179>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151849": { "content": "|<EXTRA_TOKENS_180>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151850": { "content": "|<EXTRA_TOKENS_181>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151851": { "content": "|<EXTRA_TOKENS_182>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151852": { "content": "|<EXTRA_TOKENS_183>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151853": { "content": "|<EXTRA_TOKENS_184>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151854": { "content": "|<EXTRA_TOKENS_185>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151855": { "content": "|<EXTRA_TOKENS_186>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151856": { "content": "|<EXTRA_TOKENS_187>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151857": { "content": "|<EXTRA_TOKENS_188>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151858": { "content": "|<EXTRA_TOKENS_189>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151859": { "content": "|<EXTRA_TOKENS_190>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151860": { "content": "|<EXTRA_TOKENS_191>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151861": { "content": "|<EXTRA_TOKENS_192>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151862": { "content": "|<EXTRA_TOKENS_193>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151863": { "content": "|<EXTRA_TOKENS_194>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151864": { "content": "|<EXTRA_TOKENS_195>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151865": { "content": "|<EXTRA_TOKENS_196>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151866": { "content": "|<EXTRA_TOKENS_197>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151867": { "content": "|<EXTRA_TOKENS_198>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151868": { "content": "|<EXTRA_TOKENS_199>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151869": { "content": "|<EXTRA_TOKENS_200>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151870": { "content": "|<EXTRA_TOKENS_201>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151871": { "content": "|<EXTRA_TOKENS_202>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151872": { "content": "|<EXTRA_TOKENS_203>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151873": { "content": "|<EXTRA_TOKENS_204>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151874": { "content": "|<EXTRA_TOKENS_205>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151875": { "content": "|<EXTRA_TOKENS_206>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151876": { "content": "|<EXTRA_TOKENS_207>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151877": { "content": "|<EXTRA_TOKENS_208>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151878": { "content": "|<EXTRA_TOKENS_209>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151879": { "content": "|<EXTRA_TOKENS_210>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151880": { "content": "|<EXTRA_TOKENS_211>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151881": { "content": "|<EXTRA_TOKENS_212>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151882": { "content": "|<EXTRA_TOKENS_213>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151883": { "content": "|<EXTRA_TOKENS_214>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151884": { "content": "|<EXTRA_TOKENS_215>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151885": { "content": "|<EXTRA_TOKENS_216>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151886": { "content": "|<EXTRA_TOKENS_217>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151887": { "content": "|<EXTRA_TOKENS_218>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151888": { "content": "|<EXTRA_TOKENS_219>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151889": { "content": "|<EXTRA_TOKENS_220>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151890": { "content": "|<EXTRA_TOKENS_221>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151891": { "content": "|<EXTRA_TOKENS_222>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151892": { "content": "|<EXTRA_TOKENS_223>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151893": { "content": "|<EXTRA_TOKENS_224>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151894": { "content": "|<EXTRA_TOKENS_225>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151895": { "content": "|<EXTRA_TOKENS_226>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151896": { "content": "|<EXTRA_TOKENS_227>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151897": { "content": "|<EXTRA_TOKENS_228>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151898": { "content": "|<EXTRA_TOKENS_229>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151899": { "content": "|<EXTRA_TOKENS_230>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151900": { "content": "|<EXTRA_TOKENS_231>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151901": { "content": "|<EXTRA_TOKENS_232>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151902": { "content": "|<EXTRA_TOKENS_233>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151903": { "content": "|<EXTRA_TOKENS_234>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151904": { "content": "|<EXTRA_TOKENS_235>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151905": { "content": "|<EXTRA_TOKENS_236>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151906": { "content": "|<EXTRA_TOKENS_237>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151907": { "content": "|<EXTRA_TOKENS_238>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151908": { "content": "|<EXTRA_TOKENS_239>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151909": { "content": "|<EXTRA_TOKENS_240>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151910": { "content": "|<EXTRA_TOKENS_241>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151911": { "content": "|<EXTRA_TOKENS_242>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151912": { "content": "|<EXTRA_TOKENS_243>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151913": { "content": "|<EXTRA_TOKENS_244>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151914": { "content": "|<EXTRA_TOKENS_245>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151915": { "content": "|<EXTRA_TOKENS_246>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151916": { "content": "|<EXTRA_TOKENS_247>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151917": { "content": "|<EXTRA_TOKENS_248>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151918": { "content": "|<EXTRA_TOKENS_249>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151919": { "content": "|<EXTRA_TOKENS_250>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151920": { "content": "|<EXTRA_TOKENS_251>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151921": { "content": "|<EXTRA_TOKENS_252>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151922": { "content": "|<EXTRA_TOKENS_253>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151923": { "content": "|<EXTRA_TOKENS_254>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151924": { "content": "|<EXTRA_TOKENS_255>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151925": { "content": "|<EXTRA_TOKENS_256>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151926": { "content": "|<EXTRA_TOKENS_257>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151927": { "content": "|<EXTRA_TOKENS_258>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151928": { "content": "|<EXTRA_TOKENS_259>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151929": { "content": "|<EXTRA_TOKENS_260>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151930": { "content": "|<EXTRA_TOKENS_261>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151931": { "content": "|<EXTRA_TOKENS_262>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151932": { "content": "|<EXTRA_TOKENS_263>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151933": { "content": "|<EXTRA_TOKENS_264>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151934": { "content": "|<EXTRA_TOKENS_265>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151935": { "content": "|<EXTRA_TOKENS_266>|", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151936": { "content": "<im_start>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151937": { "content": "<im_end>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151938": { "content": "<im_patch>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151939": { "content": "<im_col>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151940": { "content": "<low_res_im_start>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151941": { "content": "<|image|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151942": { "content": "<im_low>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151943": { "content": "<frame_start>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151944": { "content": "<frame_end>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "151945": { "content": "<|video|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true } }, "additional_special_tokens": [ "|<EXTRA_TOKENS_0>|", "|<EXTRA_TOKENS_1>|", "|<EXTRA_TOKENS_2>|", "|<EXTRA_TOKENS_3>|", "|<EXTRA_TOKENS_4>|", "|<EXTRA_TOKENS_5>|", "|<EXTRA_TOKENS_6>|", "|<EXTRA_TOKENS_7>|", "|<EXTRA_TOKENS_8>|", "|<EXTRA_TOKENS_9>|", "|<EXTRA_TOKENS_10>|", "|<EXTRA_TOKENS_11>|", "|<EXTRA_TOKENS_12>|", "|<EXTRA_TOKENS_13>|", "|<EXTRA_TOKENS_14>|", "|<EXTRA_TOKENS_15>|", "|<EXTRA_TOKENS_16>|", "|<EXTRA_TOKENS_17>|", "|<EXTRA_TOKENS_18>|", "|<EXTRA_TOKENS_19>|", "|<EXTRA_TOKENS_20>|", "|<EXTRA_TOKENS_21>|", "|<EXTRA_TOKENS_22>|", "|<EXTRA_TOKENS_23>|", "|<EXTRA_TOKENS_24>|", "|<EXTRA_TOKENS_25>|", "|<EXTRA_TOKENS_26>|", "|<EXTRA_TOKENS_27>|", "|<EXTRA_TOKENS_28>|", "|<EXTRA_TOKENS_29>|", "|<EXTRA_TOKENS_30>|", "|<EXTRA_TOKENS_31>|", "|<EXTRA_TOKENS_32>|", "|<EXTRA_TOKENS_33>|", "|<EXTRA_TOKENS_34>|", "|<EXTRA_TOKENS_35>|", "|<EXTRA_TOKENS_36>|", "|<EXTRA_TOKENS_37>|", "|<EXTRA_TOKENS_38>|", "|<EXTRA_TOKENS_39>|", "|<EXTRA_TOKENS_40>|", "|<EXTRA_TOKENS_41>|", "|<EXTRA_TOKENS_42>|", "|<EXTRA_TOKENS_43>|", "|<EXTRA_TOKENS_44>|", "|<EXTRA_TOKENS_45>|", "|<EXTRA_TOKENS_46>|", "|<EXTRA_TOKENS_47>|", "|<EXTRA_TOKENS_48>|", "|<EXTRA_TOKENS_49>|", "|<EXTRA_TOKENS_50>|", "|<EXTRA_TOKENS_51>|", "|<EXTRA_TOKENS_52>|", "|<EXTRA_TOKENS_53>|", "|<EXTRA_TOKENS_54>|", "|<EXTRA_TOKENS_55>|", "|<EXTRA_TOKENS_56>|", "|<EXTRA_TOKENS_57>|", "|<EXTRA_TOKENS_58>|", "|<EXTRA_TOKENS_59>|", "|<EXTRA_TOKENS_60>|", "|<EXTRA_TOKENS_61>|", "|<EXTRA_TOKENS_62>|", "|<EXTRA_TOKENS_63>|", "|<EXTRA_TOKENS_64>|", "|<EXTRA_TOKENS_65>|", "|<EXTRA_TOKENS_66>|", "|<EXTRA_TOKENS_67>|", "|<EXTRA_TOKENS_68>|", "|<EXTRA_TOKENS_69>|", "|<EXTRA_TOKENS_70>|", "|<EXTRA_TOKENS_71>|", "|<EXTRA_TOKENS_72>|", "|<EXTRA_TOKENS_73>|", "|<EXTRA_TOKENS_74>|", "|<EXTRA_TOKENS_75>|", "|<EXTRA_TOKENS_76>|", "|<EXTRA_TOKENS_77>|", "|<EXTRA_TOKENS_78>|", "|<EXTRA_TOKENS_79>|", "|<EXTRA_TOKENS_80>|", "|<EXTRA_TOKENS_81>|", "|<EXTRA_TOKENS_82>|", "|<EXTRA_TOKENS_83>|", "|<EXTRA_TOKENS_84>|", "|<EXTRA_TOKENS_85>|", "|<EXTRA_TOKENS_86>|", "|<EXTRA_TOKENS_87>|", "|<EXTRA_TOKENS_88>|", "|<EXTRA_TOKENS_89>|", "|<EXTRA_TOKENS_90>|", "|<EXTRA_TOKENS_91>|", "|<EXTRA_TOKENS_92>|", "|<EXTRA_TOKENS_93>|", "|<EXTRA_TOKENS_94>|", "|<EXTRA_TOKENS_95>|", "|<EXTRA_TOKENS_96>|", "|<EXTRA_TOKENS_97>|", "|<EXTRA_TOKENS_98>|", "|<EXTRA_TOKENS_99>|", "|<EXTRA_TOKENS_100>|", "|<EXTRA_TOKENS_101>|", "|<EXTRA_TOKENS_102>|", "|<EXTRA_TOKENS_103>|", "|<EXTRA_TOKENS_104>|", "|<EXTRA_TOKENS_105>|", "|<EXTRA_TOKENS_106>|", "|<EXTRA_TOKENS_107>|", "|<EXTRA_TOKENS_108>|", "|<EXTRA_TOKENS_109>|", "|<EXTRA_TOKENS_110>|", "|<EXTRA_TOKENS_111>|", "|<EXTRA_TOKENS_112>|", "|<EXTRA_TOKENS_113>|", "|<EXTRA_TOKENS_114>|", "|<EXTRA_TOKENS_115>|", "|<EXTRA_TOKENS_116>|", "|<EXTRA_TOKENS_117>|", "|<EXTRA_TOKENS_118>|", "|<EXTRA_TOKENS_119>|", "|<EXTRA_TOKENS_120>|", "|<EXTRA_TOKENS_121>|", "|<EXTRA_TOKENS_122>|", "|<EXTRA_TOKENS_123>|", "|<EXTRA_TOKENS_124>|", "|<EXTRA_TOKENS_125>|", "|<EXTRA_TOKENS_126>|", "|<EXTRA_TOKENS_127>|", "|<EXTRA_TOKENS_128>|", "|<EXTRA_TOKENS_129>|", "|<EXTRA_TOKENS_130>|", "|<EXTRA_TOKENS_131>|", "|<EXTRA_TOKENS_132>|", "|<EXTRA_TOKENS_133>|", "|<EXTRA_TOKENS_134>|", "|<EXTRA_TOKENS_135>|", "|<EXTRA_TOKENS_136>|", "|<EXTRA_TOKENS_137>|", "|<EXTRA_TOKENS_138>|", "|<EXTRA_TOKENS_139>|", "|<EXTRA_TOKENS_140>|", "|<EXTRA_TOKENS_141>|", "|<EXTRA_TOKENS_142>|", "|<EXTRA_TOKENS_143>|", "|<EXTRA_TOKENS_144>|", "|<EXTRA_TOKENS_145>|", "|<EXTRA_TOKENS_146>|", "|<EXTRA_TOKENS_147>|", "|<EXTRA_TOKENS_148>|", "|<EXTRA_TOKENS_149>|", "|<EXTRA_TOKENS_150>|", "|<EXTRA_TOKENS_151>|", "|<EXTRA_TOKENS_152>|", "|<EXTRA_TOKENS_153>|", "|<EXTRA_TOKENS_154>|", "|<EXTRA_TOKENS_155>|", "|<EXTRA_TOKENS_156>|", "|<EXTRA_TOKENS_157>|", "|<EXTRA_TOKENS_158>|", "|<EXTRA_TOKENS_159>|", "|<EXTRA_TOKENS_160>|", "|<EXTRA_TOKENS_161>|", "|<EXTRA_TOKENS_162>|", "|<EXTRA_TOKENS_163>|", "|<EXTRA_TOKENS_164>|", "|<EXTRA_TOKENS_165>|", "|<EXTRA_TOKENS_166>|", "|<EXTRA_TOKENS_167>|", "|<EXTRA_TOKENS_168>|", "|<EXTRA_TOKENS_169>|", "|<EXTRA_TOKENS_170>|", "|<EXTRA_TOKENS_171>|", "|<EXTRA_TOKENS_172>|", "|<EXTRA_TOKENS_173>|", "|<EXTRA_TOKENS_174>|", "|<EXTRA_TOKENS_175>|", "|<EXTRA_TOKENS_176>|", "|<EXTRA_TOKENS_177>|", "|<EXTRA_TOKENS_178>|", "|<EXTRA_TOKENS_179>|", "|<EXTRA_TOKENS_180>|", "|<EXTRA_TOKENS_181>|", "|<EXTRA_TOKENS_182>|", "|<EXTRA_TOKENS_183>|", "|<EXTRA_TOKENS_184>|", "|<EXTRA_TOKENS_185>|", "|<EXTRA_TOKENS_186>|", "|<EXTRA_TOKENS_187>|", "|<EXTRA_TOKENS_188>|", "|<EXTRA_TOKENS_189>|", "|<EXTRA_TOKENS_190>|", "|<EXTRA_TOKENS_191>|", "|<EXTRA_TOKENS_192>|", "|<EXTRA_TOKENS_193>|", "|<EXTRA_TOKENS_194>|", "|<EXTRA_TOKENS_195>|", "|<EXTRA_TOKENS_196>|", "|<EXTRA_TOKENS_197>|", "|<EXTRA_TOKENS_198>|", "|<EXTRA_TOKENS_199>|", "|<EXTRA_TOKENS_200>|", "|<EXTRA_TOKENS_201>|", "|<EXTRA_TOKENS_202>|", "|<EXTRA_TOKENS_203>|", "|<EXTRA_TOKENS_204>|", "|<EXTRA_TOKENS_205>|", "|<EXTRA_TOKENS_206>|", "|<EXTRA_TOKENS_207>|", "|<EXTRA_TOKENS_208>|", "|<EXTRA_TOKENS_209>|", "|<EXTRA_TOKENS_210>|", "|<EXTRA_TOKENS_211>|", "|<EXTRA_TOKENS_212>|", "|<EXTRA_TOKENS_213>|", "|<EXTRA_TOKENS_214>|", "|<EXTRA_TOKENS_215>|", "|<EXTRA_TOKENS_216>|", "|<EXTRA_TOKENS_217>|", "|<EXTRA_TOKENS_218>|", "|<EXTRA_TOKENS_219>|", "|<EXTRA_TOKENS_220>|", "|<EXTRA_TOKENS_221>|", "|<EXTRA_TOKENS_222>|", "|<EXTRA_TOKENS_223>|", "|<EXTRA_TOKENS_224>|", "|<EXTRA_TOKENS_225>|", "|<EXTRA_TOKENS_226>|", "|<EXTRA_TOKENS_227>|", "|<EXTRA_TOKENS_228>|", "|<EXTRA_TOKENS_229>|", "|<EXTRA_TOKENS_230>|", "|<EXTRA_TOKENS_231>|", "|<EXTRA_TOKENS_232>|", "|<EXTRA_TOKENS_233>|", "|<EXTRA_TOKENS_234>|", "|<EXTRA_TOKENS_235>|", "|<EXTRA_TOKENS_236>|", "|<EXTRA_TOKENS_237>|", "|<EXTRA_TOKENS_238>|", "|<EXTRA_TOKENS_239>|", "|<EXTRA_TOKENS_240>|", "|<EXTRA_TOKENS_241>|", "|<EXTRA_TOKENS_242>|", "|<EXTRA_TOKENS_243>|", "|<EXTRA_TOKENS_244>|", "|<EXTRA_TOKENS_245>|", "|<EXTRA_TOKENS_246>|", "|<EXTRA_TOKENS_247>|", "|<EXTRA_TOKENS_248>|", "|<EXTRA_TOKENS_249>|", "|<EXTRA_TOKENS_250>|", "|<EXTRA_TOKENS_251>|", "|<EXTRA_TOKENS_252>|", "|<EXTRA_TOKENS_253>|", "|<EXTRA_TOKENS_254>|", "|<EXTRA_TOKENS_255>|", "|<EXTRA_TOKENS_256>|", "|<EXTRA_TOKENS_257>|", "|<EXTRA_TOKENS_258>|", "|<EXTRA_TOKENS_259>|", "|<EXTRA_TOKENS_260>|", "|<EXTRA_TOKENS_261>|", "|<EXTRA_TOKENS_262>|", "|<EXTRA_TOKENS_263>|", "|<EXTRA_TOKENS_264>|", "|<EXTRA_TOKENS_265>|", "|<EXTRA_TOKENS_266>|", "<im_start>", "<im_end>", "<im_patch>", "<im_col>", "<low_res_im_start>", "<|image|>", "<im_low>", "<frame_start>", "<frame_end>", "<|video|>" ], "auto_map": { "AutoProcessor": "processing_molmo2.Molmo2Processor" }, "bos_token": "<|im_end|>", "clean_up_tokenization_spaces": false, "eos_token": "<|im_end|>", "errors": "replace", "extra_special_tokens": {}, "model_max_length": 131072, "pad_token": "<|endoftext|>", "processor_class": "Molmo2Processor", "split_special_tokens": false, "tokenizer_class": "Qwen2Tokenizer", "unk_token": null }
{
"add_bos_token": false,
"add_prefix_space": false,
"added_tokens_decoder": {
"151643": {
"content": "<|endoftext|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151644": {
"content": "<|im_start|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151645": {
"content": "<|im_end|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151646": {
"content": "<|object_ref_start|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151647": {
"content": "<|object_ref_end|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151648": {
"content": "<|box_start|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151649": {
"content": "<|box_end|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151650": {
"content": "<|quad_start|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151651": {
"content": "<|quad_end|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151652": {
"content": "<|vision_start|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151653": {
"content": "<|vision_end|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},
"151654": {
"content": "<|vision_pad|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false,
"special": true
},