Upload tokenizer

special_tokens_map.json

@@ -0,0 +1,9 @@
+{
+  "bos_token": "<cls>",
+  "cls_token": "<cls>",
+  "eos_token": "<sep>",
+  "mask_token": "<mask>",
+  "pad_token": "<pad>",
+  "sep_token": "<sep>",
+  "unk_token": "<unk>"
+}

tokenizer.json

@@ -0,0 +1,516 @@
+{
+  "version": "1.0",
+  "truncation": null,
+  "padding": null,
+  "added_tokens": [
+    {
+      "id": 0,
+      "content": "<pad>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 1,
+      "content": "<cls>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 2,
+      "content": "<sep>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 3,
+      "content": "<unk>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 4,
+      "content": "<mask>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    }
+  ],
+  "normalizer": null,
+  "pre_tokenizer": null,
+  "post_processor": {
+    "type": "TemplateProcessing",
+    "single": [
+      {
+        "SpecialToken": {
+          "id": "<cls>",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      },
+      {
+        "SpecialToken": {
+          "id": "<sep>",
+          "type_id": 0
+        }
+      }
+    ],
+    "pair": [
+      {
+        "SpecialToken": {
+          "id": "<cls>",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      },
+      {
+        "SpecialToken": {
+          "id": "<sep>",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "B",
+          "type_id": 1
+        }
+      },
+      {
+        "SpecialToken": {
+          "id": "<sep>",
+          "type_id": 1
+        }
+      }
+    ],
+    "special_tokens": {
+      "<cls>": {
+        "id": "<cls>",
+        "ids": [
+          1
+        ],
+        "tokens": [
+          "<cls>"
+        ]
+      },
+      "<sep>": {
+        "id": "<sep>",
+        "ids": [
+          2
+        ],
+        "tokens": [
+          "<sep>"
+        ]
+      }
+    }
+  },
+  "decoder": {
+    "type": "BPEDecoder",
+    "suffix": "</w>"
+  },
+  "model": {
+    "type": "BPE",
+    "dropout": null,
+    "unk_token": null,
+    "continuing_subword_prefix": null,
+    "end_of_word_suffix": null,
+    "fuse_unk": false,
+    "byte_fallback": false,
+    "vocab": {
+      "<pad>": 0,
+      "<cls>": 1,
+      "<sep>": 2,
+      "<unk>": 3,
+      "<mask>": 4,
+      "A": 5,
+      "B": 6,
+      "C": 7,
+      "D": 8,
+      "E": 9,
+      "F": 10,
+      "G": 11,
+      "H": 12,
+      "I": 13,
+      "K": 14,
+      "L": 15,
+      "M": 16,
+      "N": 17,
+      "O": 18,
+      "P": 19,
+      "Q": 20,
+      "R": 21,
+      "S": 22,
+      "T": 23,
+      "U": 24,
+      "V": 25,
+      "W": 26,
+      "X": 27,
+      "Y": 28,
+      "Z": 29,
+      "AA": 30,
+      "LL": 31,
+      "SS": 32,
+      "LA": 33,
+      "LS": 34,
+      "GG": 35,
+      "LV": 36,
+      "LR": 37,
+      "LE": 38,
+      "LG": 39,
+      "LT": 40,
+      "AV": 41,
+      "AG": 42,
+      "LP": 43,
+      "LD": 44,
+      "LI": 45,
+      "AS": 46,
+      "EE": 47,
+      "RR": 48,
+      "LK": 49,
+      "VV": 50,
+      "AT": 51,
+      "AE": 52,
+      "SG": 53,
+      "DG": 54,
+      "AR": 55,
+      "KK": 56,
+      "AI": 57,
+      "AP": 58,
+      "TT": 59,
+      "LN": 60,
+      "LF": 61,
+      "LQ": 62,
+      "AD": 63,
+      "SV": 64,
+      "ST": 65,
+      "SI": 66,
+      "SP": 67,
+      "DE": 68,
+      "TG": 69,
+      "TV": 70,
+      "SR": 71,
+      "SE": 72,
+      "DV": 73,
+      "PP": 74,
+      "KE": 75,
+      "KI": 76,
+      "DI": 77,
+      "GV": 78,
+      "AQ": 79,
+      "AF": 80,
+      "RV": 81,
+      "TI": 82,
+      "NI": 83,
+      "PV": 84,
+      "RE": 85,
+      "SF": 86,
+      "KV": 87,
+      "NG": 88,
+      "LY": 89,
+      "RI": 90,
+      "PG": 91,
+      "RG": 92,
+      "DD": 93,
+      "PE": 94,
+      "AK": 95,
+      "NV": 96,
+      "SD": 97,
+      "TE": 98,
+      "QQ": 99,
+      "NE": 100,
+      "SK": 101,
+      "LH": 102,
+      "SQ": 103,
+      "GI": 104,
+      "FV": 105,
+      "TP": 106,
+      "GE": 107,
+      "SN": 108,
+      "FI": 109,
+      "AY": 110,
+      "AN": 111,
+      "QV": 112,
+      "RK": 113,
+      "SY": 114,
+      "IV": 115,
+      "FE": 116,
+      "DP": 117,
+      "RP": 118,
+      "IE": 119,
+      "QE": 120,
+      "FG": 121,
+      "LM": 122,
+      "NN": 123,
+      "KG": 124,
+      "RT": 125,
+      "RD": 126,
+      "RQ": 127,
+      "II": 128,
+      "TD": 129,
+      "AH": 130,
+      "AM": 131,
+      "YV": 132,
+      "RF": 133,
+      "EV": 134,
+      "KN": 135,
+      "KD": 136,
+      "IG": 137,
+      "LC": 138,
+      "KT": 139,
+      "KP": 140,
+      "RY": 141,
+      "SH": 142,
+      "QG": 143,
+      "NP": 144,
+      "QI": 145,
+      "FD": 146,
+      "YG": 147,
+      "RN": 148,
+      "YE": 149,
+      "TF": 150,
+      "MV": 151,
+      "RH": 152,
+      "LW": 153,
+      "QP": 154,
+      "TN": 155,
+      "DF": 156,
+      "KQ": 157,
+      "YI": 158,
+      "ME": 159,
+      "VE": 160,
+      "LAA": 161,
+      "PD": 162,
+      "FF": 163,
+      "PI": 164,
+      "AC": 165,
+      "ND": 166,
+      "VG": 167,
+      "TY": 168,
+      "TQ": 169,
+      "SM": 170,
+      "AW": 171,
+      "KY": 172,
+      "HG": 173,
+      "SC": 174,
+      "HV": 175,
+      "ID": 176,
+      "KF": 177,
+      "EG": 178,
+      "KR": 179,
+      "EI": 180,
+      "MG": 181,
+      "TR": 182,
+      "NF": 183,
+      "SW": 184,
+      "TK": 185,
+      "YD": 186,
+      "NQ": 187,
+      "ALL": 188,
+      "MI": 189,
+      "PT": 190,
+      "NY": 191,
+      "QD": 192,
+      "GD": 193,
+      "NT": 194,
+      "HE": 195,
+      "VI": 196,
+      "PF": 197,
+      "ALA": 198,
+      "RS": 199
+    },
+    "merges": [
+      "A A",
+      "L L",
+      "S S",
+      "L A",
+      "L S",
+      "G G",
+      "L V",
+      "L R",
+      "L E",
+      "L G",
+      "L T",
+      "A V",
+      "A G",
+      "L P",
+      "L D",
+      "L I",
+      "A S",
+      "E E",
+      "R R",
+      "L K",
+      "V V",
+      "A T",
+      "A E",
+      "S G",
+      "D G",
+      "A R",
+      "K K",
+      "A I",
+      "A P",
+      "T T",
+      "L N",
+      "L F",
+      "L Q",
+      "A D",
+      "S V",
+      "S T",
+      "S I",
+      "S P",
+      "D E",
+      "T G",
+      "T V",
+      "S R",
+      "S E",
+      "D V",
+      "P P",
+      "K E",
+      "K I",
+      "D I",
+      "G V",
+      "A Q",
+      "A F",
+      "R V",
+      "T I",
+      "N I",
+      "P V",
+      "R E",
+      "S F",
+      "K V",
+      "N G",
+      "L Y",
+      "R I",
+      "P G",
+      "R G",
+      "D D",
+      "P E",
+      "A K",
+      "N V",
+      "S D",
+      "T E",
+      "Q Q",
+      "N E",
+      "S K",
+      "L H",
+      "S Q",
+      "G I",
+      "F V",
+      "T P",
+      "G E",
+      "S N",
+      "F I",
+      "A Y",
+      "A N",
+      "Q V",
+      "R K",
+      "S Y",
+      "I V",
+      "F E",
+      "D P",
+      "R P",
+      "I E",
+      "Q E",
+      "F G",
+      "L M",
+      "N N",
+      "K G",
+      "R T",
+      "R D",
+      "R Q",
+      "I I",
+      "T D",
+      "A H",
+      "A M",
+      "Y V",
+      "R F",
+      "E V",
+      "K N",
+      "K D",
+      "I G",
+      "L C",
+      "K T",
+      "K P",
+      "R Y",
+      "S H",
+      "Q G",
+      "N P",
+      "Q I",
+      "F D",
+      "Y G",
+      "R N",
+      "Y E",
+      "T F",
+      "M V",
+      "R H",
+      "L W",
+      "Q P",
+      "T N",
+      "D F",
+      "K Q",
+      "Y I",
+      "M E",
+      "V E",
+      "L AA",
+      "P D",
+      "F F",
+      "P I",
+      "A C",
+      "N D",
+      "V G",
+      "T Y",
+      "T Q",
+      "S M",
+      "A W",
+      "K Y",
+      "H G",
+      "S C",
+      "H V",
+      "I D",
+      "K F",
+      "E G",
+      "K R",
+      "E I",
+      "M G",
+      "T R",
+      "N F",
+      "S W",
+      "T K",
+      "Y D",
+      "N Q",
+      "A LL",
+      "M I",
+      "P T",
+      "N Y",
+      "Q D",
+      "G D",
+      "N T",
+      "H E",
+      "V I",
+      "P F",
+      "A LA",
+      "R S"
+    ]
+  }
+}

tokenizer_config.json

@@ -0,0 +1,5 @@
+{
+  "clean_up_tokenization_spaces": true,
+  "model_max_length": 1000000000000000019884624838656,
+  "tokenizer_class": "PreTrainedTokenizerFast"
+}