Upload tokenizer.json with huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

tokenizer.json

@@ -1,176 +1,3 @@
-{
-  "version": "1.0",
-  "truncation": null,
-  "padding": null,
-  "added_tokens": [],
-  "normalizer": null,
-  "pre_tokenizer": {
-    "type": "ByteLevel",
-    "add_prefix_space": false,
-    "trim_offsets": true,
-    "use_regex": true
-  },
-  "post_processor": null,
-  "decoder": {
-    "type": "ByteLevel",
-    "add_prefix_space": true,
-    "trim_offsets": true,
-    "use_regex": true
-  },
-  "model": {
-    "type": "WordLevel",
-    "vocab": {
-      "<unk>": 0,
-      "<|endoftext|>": 1,
-      "<|im_start|>": 2,
-      "<|im_end|>": 3,
-      "<|object_ref_start|>": 4,
-      "<|object_ref_end|>": 5,
-      "<|box_start|>": 6,
-      "<|box_end|>": 7,
-      "<|quad_start|>": 8,
-      "<|quad_end|>": 9,
-      "<|vision_start|>": 10,
-      "<|vision_end|>": 11,
-      "<|vision_pad|>": 12,
-      "<|image_pad|>": 13,
-      "<|video_pad|>": 14,
-      "<tool_call>": 15,
-      "</tool_call>": 16,
-      "<|fim_prefix|>": 17,
-      "<|fim_middle|>": 18,
-      "<|fim_suffix|>": 19,
-      "<|fim_pad|>": 20,
-      "<|repo_name|>": 21,
-      "<|file_sep|>": 22,
-      "<image>": 23,
-      "</image>": 24,
-      "<ref>": 25,
-      "</ref>": 26,
-      "<box>": 27,
-      "</box>": 28,
-      "<quad>": 29,
-      "</quad>": 30,
-      "<point>": 31,
-      "</point>": 32,
-      "<slice>": 33,
-      "</slice>": 34,
-      "<image_id>": 35,
-      "</image_id>": 36,
-      "<unit>": 37,
-      "</unit>": 38,
-      "<asr>": 39,
-      "</asr>": 40,
-      "<query>": 41,
-      "</query>": 42,
-      "<|audio_start|>": 43,
-      "<|audio|>": 44,
-      "<|audio_end|>": 45,
-      "<|spk_bos|>": 46,
-      "<|spk|>": 47,
-      "<|spk_eos|>": 48,
-      "<|tts_bos|>": 49,
-      "<|tts_eos|>": 50,
-      "<|listen|>": 51,
-      "<|speak|>": 52,
-      "<|interrupt|>": 53,
-      "<|vad_start|>": 54,
-      "<|vad_end|>": 55,
-      "<reserved_43>": 56,
-      "<reserved_53>": 57,
-      "!": 58,
-      "\"": 59,
-      "#": 60,
-      "$": 61,
-      "%": 62,
-      "&": 63,
-      "'": 64,
-      "(": 65,
-      ")": 66,
-      "*": 67,
-      "+": 68,
-      ",": 69,
-      "-": 70,
-      ".": 71,
-      "/": 72,
-      "0": 73,
-      "1": 74,
-      "2": 75,
-      "3": 76,
-      "4": 77,
-      "5": 78,
-      "6": 79,
-      "7": 80,
-      "8": 81,
-      "9": 82,
-      ":": 83,
-      ";": 84,
-      "<": 85,
-      "=": 86,
-      ">": 87,
-      "?": 88,
-      "@": 89,
-      "A": 90,
-      "B": 91,
-      "C": 92,
-      "D": 93,
-      "E": 94,
-      "F": 95,
-      "G": 96,
-      "H": 97,
-      "I": 98,
-      "J": 99,
-      "K": 100,
-      "L": 101,
-      "M": 102,
-      "N": 103,
-      "O": 104,
-      "P": 105,
-      "Q": 106,
-      "R": 107,
-      "S": 108,
-      "T": 109,
-      "U": 110,
-      "V": 111,
-      "W": 112,
-      "X": 113,
-      "Y": 114,
-      "Z": 115,
-      "[": 116,
-      "\\": 117,
-      "]": 118,
-      "^": 119,
-      "_": 120,
-      "`": 121,
-      "a": 122,
-      "b": 123,
-      "c": 124,
-      "d": 125,
-      "e": 126,
-      "f": 127,
-      "g": 128,
-      "h": 129,
-      "i": 130,
-      "j": 131,
-      "k": 132,
-      "l": 133,
-      "m": 134,
-      "n": 135,
-      "o": 136,
-      "p": 137,
-      "q": 138,
-      "r": 139,
-      "s": 140,
-      "t": 141,
-      "u": 142,
-      "v": 143,
-      "w": 144,
-      "x": 145,
-      "y": 146,
-      "z": 147,
-      "{": 148,
-      "|": 149
-    },
-    "unk_token": "<unk>"
-  }
-}
+version https://git-lfs.github.com/spec/v1
+oid sha256:e43057e8380937b8edf2c9ea6ad34a3875d89c390e97024dfac307f3a4a11321
+size 11428583