Upload tokenizer

special_tokens_map.json

@@ -0,0 +1,7 @@
+{
+  "cls_token": "[CLS]",
+  "mask_token": "[MASK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "unk_token": "[UNK]"
+}

tokenizer.json

@@ -0,0 +1,450 @@
+{
+  "version": "1.0",
+  "truncation": null,
+  "padding": null,
+  "added_tokens": [
+    {
+      "id": 0,
+      "content": "[PAD]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 1,
+      "content": "[UNK]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 2,
+      "content": "[CLS]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 3,
+      "content": "[SEP]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 4,
+      "content": "[MASK]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    }
+  ],
+  "normalizer": {
+    "type": "BertNormalizer",
+    "clean_text": true,
+    "handle_chinese_chars": true,
+    "strip_accents": null,
+    "lowercase": true
+  },
+  "pre_tokenizer": {
+    "type": "BertPreTokenizer"
+  },
+  "post_processor": {
+    "type": "TemplateProcessing",
+    "single": [
+      {
+        "SpecialToken": {
+          "id": "[CLS]",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      },
+      {
+        "SpecialToken": {
+          "id": "[SEP]",
+          "type_id": 0
+        }
+      }
+    ],
+    "pair": [
+      {
+        "SpecialToken": {
+          "id": "[CLS]",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "A",
+          "type_id": 0
+        }
+      },
+      {
+        "SpecialToken": {
+          "id": "[SEP]",
+          "type_id": 0
+        }
+      },
+      {
+        "Sequence": {
+          "id": "B",
+          "type_id": 1
+        }
+      },
+      {
+        "SpecialToken": {
+          "id": "[SEP]",
+          "type_id": 1
+        }
+      }
+    ],
+    "special_tokens": {
+      "[CLS]": {
+        "id": "[CLS]",
+        "ids": [
+          2
+        ],
+        "tokens": [
+          "[CLS]"
+        ]
+      },
+      "[SEP]": {
+        "id": "[SEP]",
+        "ids": [
+          3
+        ],
+        "tokens": [
+          "[SEP]"
+        ]
+      }
+    }
+  },
+  "decoder": {
+    "type": "WordPiece",
+    "prefix": "##",
+    "cleanup": true
+  },
+  "model": {
+    "type": "WordPiece",
+    "unk_token": "[UNK]",
+    "continuing_subword_prefix": "##",
+    "max_input_chars_per_word": 100,
+    "vocab": {
+      "[PAD]": 0,
+      "[UNK]": 1,
+      "[CLS]": 2,
+      "[SEP]": 3,
+      "[MASK]": 4,
+      "!": 5,
+      ",": 6,
+      ".": 7,
+      "?": 8,
+      "a": 9,
+      "b": 10,
+      "c": 11,
+      "d": 12,
+      "e": 13,
+      "f": 14,
+      "g": 15,
+      "h": 16,
+      "i": 17,
+      "k": 18,
+      "l": 19,
+      "m": 20,
+      "n": 21,
+      "o": 22,
+      "p": 23,
+      "q": 24,
+      "r": 25,
+      "s": 26,
+      "t": 27,
+      "u": 28,
+      "v": 29,
+      "w": 30,
+      "x": 31,
+      "y": 32,
+      "z": 33,
+      "##o": 34,
+      "##r": 35,
+      "##g": 36,
+      "##t": 37,
+      "##i": 38,
+      "##l": 39,
+      "##e": 40,
+      "##s": 41,
+      "##a": 42,
+      "##n": 43,
+      "##u": 44,
+      "##c": 45,
+      "##h": 46,
+      "##v": 47,
+      "##y": 48,
+      "##k": 49,
+      "##p": 50,
+      "##w": 51,
+      "##d": 52,
+      "##m": 53,
+      "##b": 54,
+      "##f": 55,
+      "##x": 56,
+      "##z": 57,
+      "th": 58,
+      "##ou": 59,
+      "##re": 60,
+      "the": 61,
+      "##nd": 62,
+      "##is": 63,
+      "##es": 64,
+      "##er": 65,
+      "my": 66,
+      "##or": 67,
+      "##ve": 68,
+      "ha": 69,
+      "##it": 70,
+      "##ll": 71,
+      "to": 72,
+      "##nt": 73,
+      "and": 74,
+      "no": 75,
+      "##ed": 76,
+      "mo": 77,
+      "##st": 78,
+      "##at": 79,
+      "in": 80,
+      "thou": 81,
+      "##in": 82,
+      "##ea": 83,
+      "##me": 84,
+      "co": 85,
+      "of": 86,
+      "##ir": 87,
+      "wh": 88,
+      "##on": 89,
+      "##el": 90,
+      "not": 91,
+      "is": 92,
+      "wi": 93,
+      "##ee": 94,
+      "as": 95,
+      "##ra": 96,
+      "##th": 97,
+      "##ld": 98,
+      "most": 99,
+      "for": 100,
+      "##ri": 101,
+      "will": 102,
+      "be": 103,
+      "ca": 104,
+      "me": 105,
+      "so": 106,
+      "sh": 107,
+      "##ow": 108,
+      "##il": 109,
+      "##en": 110,
+      "##se": 111,
+      "##ch": 112,
+      "thy": 113,
+      "##est": 114,
+      "have": 115,
+      "what": 116,
+      "are": 117,
+      "it": 118,
+      "li": 119,
+      "sp": 120,
+      "you": 121,
+      "##oo": 122,
+      "##ty": 123,
+      "##la": 124,
+      "##ar": 125,
+      "##ay": 126,
+      "##ke": 127,
+      "this": 128,
+      "thee": 129,
+      "##ear": 130,
+      "come": 131,
+      "##irit": 132,
+      "spirit": 133,
+      "ch": 134,
+      "do": 135,
+      "his": 136,
+      "##ro": 137,
+      "##le": 138,
+      "##ly": 139,
+      "##ut": 140,
+      "##ith": 141,
+      "more": 142,
+      "##ment": 143,
+      "all": 144,
+      "ba": 145,
+      "but": 146,
+      "de": 147,
+      "lo": 148,
+      "st": 149,
+      "see": 150,
+      "we": 151,
+      "wor": 152,
+      "with": 153,
+      "##gh": 154,
+      "##ic": 155,
+      "##an": 156,
+      "##ake": 157,
+      "##ul": 158,
+      "##mp": 159,
+      "##ber": 160,
+      "that": 161,
+      "##ould": 162,
+      "##ist": 163,
+      "now": 164,
+      "##ing": 165,
+      "##eep": 166,
+      "am": 167,
+      "ari": 168,
+      "ex": 169,
+      "go": 170,
+      "ho": 171,
+      "he": 172,
+      "per": 173,
+      "re": 174,
+      "sou": 175,
+      "ser": 176,
+      "sla": 177,
+      "say": 178,
+      "tis": 179,
+      "wa": 180,
+      "##ge": 181,
+      "##id": 182,
+      "##et": 183,
+      "##ement": 184,
+      "##ne": 185,
+      "##nst": 186,
+      "##ure": 187,
+      "##ct": 188,
+      "##ves": 189,
+      "##ms": 190,
+      "##med": 191,
+      "##for": 192,
+      "##ful": 193,
+      "##our": 194,
+      "##ous": 195,
+      "they": 196,
+      "##ish": 197,
+      "##ess": 198,
+      "##nter": 199,
+      "##eas": 200,
+      "liber": 201,
+      "##ood": 202,
+      "char": 203,
+      "##rom": 204,
+      "ariel": 205,
+      "serv": 206,
+      "slave": 207,
+      "liberty": 208,
+      "ad": 209,
+      "bo": 210,
+      "br": 211,
+      "by": 212,
+      "bes": 213,
+      "bra": 214,
+      "bear": 215,
+      "du": 216,
+      "ear": 217,
+      "fre": 218,
+      "fir": 219,
+      "fri": 220,
+      "fly": 221,
+      "fet": 222,
+      "from": 223,
+      "gre": 224,
+      "gra": 225,
+      "good": 226,
+      "hon": 227,
+      "how": 228,
+      "kn": 229,
+      "king": 230,
+      "make": 231,
+      "mist": 232,
+      "ne": 233,
+      "on": 234,
+      "ou": 235,
+      "ow": 236,
+      "po": 237,
+      "pl": 238,
+      "qu": 239,
+      "rel": 240,
+      "sl": 241,
+      "sa": 242,
+      "su": 243,
+      "sw": 244,
+      "sen": 245,
+      "tr": 246,
+      "would": 247,
+      "wish": 248,
+      "##ore": 249,
+      "##te": 250,
+      "##tu": 251,
+      "##ip": 252,
+      "##ib": 253,
+      "##ion": 254,
+      "##igh": 255,
+      "##lib": 256,
+      "##ere": 257,
+      "##end": 258,
+      "##ss": 259,
+      "##ses": 260,
+      "##and": 261,
+      "##all": 262,
+      "##ant": 263,
+      "##ain": 264,
+      "##ue": 265,
+      "##us": 266,
+      "##ck": 267,
+      "##cit": 268,
+      "##ver": 269,
+      "##vil": 270,
+      "##fe": 271,
+      "##ress": 272,
+      "them": 273,
+      "there": 274,
+      "##nds": 275,
+      "##ven": 276,
+      "has": 277,
+      "hast": 278,
+      "hath": 279,
+      "##ease": 280,
+      "cont": 281,
+      "comp": 282,
+      "##one": 283,
+      "##elf": 284,
+      "forth": 285,
+      "canst": 286,
+      "calib": 287,
+      "mere": 288,
+      "ship": 289,
+      "shall": 290,
+      "##own": 291,
+      "like": 292,
+      "your": 293,
+      "##lete": 294,
+      "bad": 295,
+      "devil": 296,
+      "love": 297,
+      "seek": 298,
+      "work": 299
+    }
+  }
+}

tokenizer_config.json

@@ -0,0 +1,13 @@
+{
+  "cls_token": "[CLS]",
+  "do_lower_case": true,
+  "mask_token": "[MASK]",
+  "model_max_length": 512,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "special_tokens_map_file": null,
+  "strip_accents": null,
+  "tokenize_chinese_chars": true,
+  "tokenizer_class": "BertTokenizer",
+  "unk_token": "[UNK]"
+}

vocab.txt

@@ -0,0 +1,300 @@
+[PAD]
+[UNK]
+[CLS]
+[SEP]
+[MASK]
+!
+,
+.
+?
+a
+b
+c
+d
+e
+f
+g
+h
+i
+k
+l
+m
+n
+o
+p
+q
+r
+s
+t
+u
+v
+w
+x
+y
+z
+##o
+##r
+##g
+##t
+##i
+##l
+##e
+##s
+##a
+##n
+##u
+##c
+##h
+##v
+##y
+##k
+##p
+##w
+##d
+##m
+##b
+##f
+##x
+##z
+th
+##ou
+##re
+the
+##nd
+##is
+##es
+##er
+my
+##or
+##ve
+ha
+##it
+##ll
+to
+##nt
+and
+no
+##ed
+mo
+##st
+##at
+in
+thou
+##in
+##ea
+##me
+co
+of
+##ir
+wh
+##on
+##el
+not
+is
+wi
+##ee
+as
+##ra
+##th
+##ld
+most
+for
+##ri
+will
+be
+ca
+me
+so
+sh
+##ow
+##il
+##en
+##se
+##ch
+thy
+##est
+have
+what
+are
+it
+li
+sp
+you
+##oo
+##ty
+##la
+##ar
+##ay
+##ke
+this
+thee
+##ear
+come
+##irit
+spirit
+ch
+do
+his
+##ro
+##le
+##ly
+##ut
+##ith
+more
+##ment
+all
+ba
+but
+de
+lo
+st
+see
+we
+wor
+with
+##gh
+##ic
+##an
+##ake
+##ul
+##mp
+##ber
+that
+##ould
+##ist
+now
+##ing
+##eep
+am
+ari
+ex
+go
+ho
+he
+per
+re
+sou
+ser
+sla
+say
+tis
+wa
+##ge
+##id
+##et
+##ement
+##ne
+##nst
+##ure
+##ct
+##ves
+##ms
+##med
+##for
+##ful
+##our
+##ous
+they
+##ish
+##ess
+##nter
+##eas
+liber
+##ood
+char
+##rom
+ariel
+serv
+slave
+liberty
+ad
+bo
+br
+by
+bes
+bra
+bear
+du
+ear
+fre
+fir
+fri
+fly
+fet
+from
+gre
+gra
+good
+hon
+how
+kn
+king
+make
+mist
+ne
+on
+ou
+ow
+po
+pl
+qu
+rel
+sl
+sa
+su
+sw
+sen
+tr
+would
+wish
+##ore
+##te
+##tu
+##ip
+##ib
+##ion
+##igh
+##lib
+##ere
+##end
+##ss
+##ses
+##and
+##all
+##ant
+##ain
+##ue
+##us
+##ck
+##cit
+##ver
+##vil
+##fe
+##ress
+them
+there
+##nds
+##ven
+has
+hast
+hath
+##ease
+cont
+comp
+##one
+##elf
+forth
+canst
+calib
+mere
+ship
+shall
+##own
+like
+your
+##lete
+bad
+devil
+love
+seek
+work