Upload tokenizer

.gitattributes

@@ -32,3 +32,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

added_tokens.json

@@ -0,0 +1,32 @@
+{
+  "ay": 250127,
+  "bn": 250121,
+  "bs": 250120,
+  "cs": 250124,
+  "eo": 250119,
+  "eu": 250123,
+  "gd": 250103,
+  "gn": 250109,
+  "jv": 250104,
+  "kn": 250105,
+  "lg": 250114,
+  "mk": 250100,
+  "mn": 250116,
+  "mr": 250112,
+  "ms": 250111,
+  "nl": 250107,
+  "ps": 250113,
+  "sd": 250110,
+  "sl": 250129,
+  "so": 250102,
+  "sq": 250128,
+  "sr": 250122,
+  "sv": 250125,
+  "sw": 250117,
+  "tk": 250108,
+  "tl": 250115,
+  "ts": 250126,
+  "yi": 250101,
+  "zh-CN": 250106,
+  "zh-TW": 250118
+}

special_tokens_map.json

@@ -0,0 +1,5 @@
+{
+  "eos_token": "</s>",
+  "pad_token": "<pad>",
+  "unk_token": "<unk>"
+}

spiece.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef78f86560d809067d12bac6c09f19a462cb3af3f54d2b8acbba26e1433125d6
+size 4309802

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:db723bb2cbad20506465e5c75220514c9280a8718792fc7d2f39aa9af04e4caf
+size 16332610

tokenizer_config.json

@@ -0,0 +1,13 @@
+{
+  "additional_special_tokens": null,
+  "eos_token": "</s>",
+  "extra_ids": 0,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "return_tensors": "pt",
+  "sp_model_kwargs": {},
+  "special_tokens_map_file": "C:\\Users\\alek/.cache\\huggingface\\hub\\models--google--mt5-small\\snapshots\\38f23af8ec210eb6c376d40e9c56bd25a80f195d\\special_tokens_map.json",
+  "tokenizer_class": "T5Tokenizer",
+  "unk_token": "<unk>",
+  "use_fast": true
+}