Upload tokenizer

tokenizer.json

@@ -179,30 +179,30 @@
       "x": 31,
       "y": 32,
       "z": 33,
-      "##o": 34,
-      "##r": 35,
-      "##g": 36,
-      "##t": 37,
-      "##i": 38,
-      "##l": 39,
-      "##e": 40,
-      "##s": 41,
-      "##a": 42,
-      "##n": 43,
-      "##u": 44,
-      "##c": 45,
-      "##h": 46,
-      "##v": 47,
-      "##y": 48,
-      "##k": 49,
-      "##p": 50,
-      "##w": 51,
-      "##d": 52,
-      "##m": 53,
-      "##b": 54,
-      "##f": 55,
-      "##x": 56,
-      "##z": 57,
+      "##l": 34,
+      "##e": 35,
+      "##p": 36,
+      "##i": 37,
+      "##s": 38,
+      "##t": 39,
+      "##r": 40,
+      "##f": 41,
+      "##w": 42,
+      "##a": 43,
+      "##o": 44,
+      "##u": 45,
+      "##n": 46,
+      "##d": 47,
+      "##g": 48,
+      "##h": 49,
+      "##v": 50,
+      "##k": 51,
+      "##m": 52,
+      "##z": 53,
+      "##y": 54,
+      "##c": 55,
+      "##b": 56,
+      "##x": 57,
       "th": 58,
       "##ou": 59,
       "##re": 60,
@@ -215,8 +215,8 @@
       "##or": 67,
       "##ve": 68,
       "ha": 69,
-      "##it": 70,
-      "##ll": 71,
+      "##ll": 70,
+      "##it": 71,
       "to": 72,
       "##nt": 73,
       "and": 74,
@@ -227,23 +227,23 @@
       "##at": 79,
       "in": 80,
       "thou": 81,
-      "##in": 82,
-      "##ea": 83,
+      "##ea": 82,
+      "##in": 83,
       "##me": 84,
       "co": 85,
       "of": 86,
       "##ir": 87,
       "wh": 88,
-      "##on": 89,
-      "##el": 90,
+      "##el": 89,
+      "##on": 90,
       "not": 91,
       "is": 92,
       "wi": 93,
       "##ee": 94,
       "as": 95,
-      "##ra": 96,
+      "##ld": 96,
       "##th": 97,
-      "##ld": 98,
+      "##ra": 98,
       "most": 99,
       "for": 100,
       "##ri": 101,
@@ -253,23 +253,23 @@
       "me": 105,
       "so": 106,
       "sh": 107,
-      "##ow": 108,
-      "##il": 109,
-      "##en": 110,
-      "##se": 111,
+      "##la": 108,
+      "##en": 109,
+      "##se": 110,
+      "##ro": 111,
       "##ch": 112,
       "thy": 113,
       "##est": 114,
       "have": 115,
       "what": 116,
       "are": 117,
-      "it": 118,
-      "li": 119,
-      "sp": 120,
-      "you": 121,
-      "##oo": 122,
-      "##ty": 123,
-      "##la": 124,
+      "do": 118,
+      "it": 119,
+      "li": 120,
+      "sp": 121,
+      "you": 122,
+      "##il": 123,
+      "##ty": 124,
       "##ar": 125,
       "##ay": 126,
       "##ke": 127,
@@ -280,44 +280,44 @@
       "##irit": 132,
       "spirit": 133,
       "ch": 134,
-      "do": 135,
-      "his": 136,
-      "##ro": 137,
-      "##le": 138,
-      "##ly": 139,
-      "##ut": 140,
-      "##ith": 141,
-      "more": 142,
-      "##ment": 143,
-      "all": 144,
-      "ba": 145,
-      "but": 146,
-      "de": 147,
-      "lo": 148,
+      "go": 135,
+      "ho": 136,
+      "his": 137,
+      "lo": 138,
+      "##le": 139,
+      "##ly": 140,
+      "##ut": 141,
+      "##ith": 142,
+      "more": 143,
+      "##ment": 144,
+      "all": 145,
+      "ba": 146,
+      "but": 147,
+      "de": 148,
       "st": 149,
       "see": 150,
       "we": 151,
       "wor": 152,
       "with": 153,
-      "##gh": 154,
-      "##ic": 155,
+      "##ic": 154,
+      "##wn": 155,
       "##an": 156,
       "##ake": 157,
       "##ul": 158,
-      "##mp": 159,
-      "##ber": 160,
-      "that": 161,
-      "##ould": 162,
-      "##ist": 163,
-      "now": 164,
-      "##ing": 165,
-      "##eep": 166,
-      "am": 167,
-      "ari": 168,
-      "ex": 169,
-      "go": 170,
-      "ho": 171,
-      "he": 172,
+      "##gh": 159,
+      "##mp": 160,
+      "##ber": 161,
+      "that": 162,
+      "##ould": 163,
+      "##ist": 164,
+      "now": 165,
+      "##ing": 166,
+      "##eep": 167,
+      "am": 168,
+      "ari": 169,
+      "ex": 170,
+      "he": 171,
+      "po": 172,
       "per": 173,
       "re": 174,
       "sou": 175,
@@ -326,97 +326,97 @@
       "say": 178,
       "tis": 179,
       "wa": 180,
-      "##ge": 181,
-      "##id": 182,
-      "##et": 183,
-      "##ement": 184,
-      "##ne": 185,
-      "##nst": 186,
-      "##ure": 187,
-      "##ct": 188,
-      "##ves": 189,
-      "##ms": 190,
-      "##med": 191,
-      "##for": 192,
-      "##ful": 193,
-      "##our": 194,
+      "##et": 181,
+      "##ement": 182,
+      "##id": 183,
+      "##for": 184,
+      "##ful": 185,
+      "##od": 186,
+      "##un": 187,
+      "##ure": 188,
+      "##nst": 189,
+      "##ge": 190,
+      "##ves": 191,
+      "##ms": 192,
+      "##med": 193,
+      "##ct": 194,
       "##ous": 195,
-      "they": 196,
-      "##ish": 197,
-      "##ess": 198,
-      "##nter": 199,
-      "##eas": 200,
-      "liber": 201,
-      "##ood": 202,
-      "char": 203,
-      "##rom": 204,
+      "##our": 196,
+      "they": 197,
+      "##ish": 198,
+      "##ess": 199,
+      "##nter": 200,
+      "##eas": 201,
+      "##rom": 202,
+      "liber": 203,
+      "char": 204,
       "ariel": 205,
       "serv": 206,
       "slave": 207,
       "liberty": 208,
       "ad": 209,
-      "bo": 210,
-      "br": 211,
+      "br": 210,
+      "bo": 211,
       "by": 212,
       "bes": 213,
       "bra": 214,
       "bear": 215,
       "du": 216,
-      "ear": 217,
-      "fre": 218,
-      "fir": 219,
-      "fri": 220,
-      "fly": 221,
-      "fet": 222,
-      "from": 223,
-      "gre": 224,
-      "gra": 225,
-      "good": 226,
+      "en": 217,
+      "ear": 218,
+      "fre": 219,
+      "fir": 220,
+      "fri": 221,
+      "fly": 222,
+      "fet": 223,
+      "from": 224,
+      "gre": 225,
+      "gra": 226,
       "hon": 227,
-      "how": 228,
-      "kn": 229,
-      "king": 230,
-      "make": 231,
-      "mist": 232,
-      "ne": 233,
+      "kn": 228,
+      "king": 229,
+      "make": 230,
+      "mist": 231,
+      "ne": 232,
+      "ou": 233,
       "on": 234,
-      "ou": 235,
-      "ow": 236,
-      "po": 237,
-      "pl": 238,
-      "qu": 239,
-      "rel": 240,
-      "sl": 241,
-      "sa": 242,
-      "su": 243,
-      "sw": 244,
-      "sen": 245,
-      "tr": 246,
-      "would": 247,
-      "wish": 248,
-      "##ore": 249,
-      "##te": 250,
-      "##tu": 251,
-      "##ip": 252,
-      "##ib": 253,
-      "##ion": 254,
-      "##igh": 255,
-      "##lib": 256,
-      "##ere": 257,
-      "##end": 258,
-      "##ss": 259,
-      "##ses": 260,
-      "##and": 261,
-      "##all": 262,
-      "##ant": 263,
-      "##ain": 264,
+      "own": 235,
+      "pl": 236,
+      "qu": 237,
+      "rel": 238,
+      "sl": 239,
+      "sw": 240,
+      "sa": 241,
+      "su": 242,
+      "sen": 243,
+      "vi": 244,
+      "would": 245,
+      "wish": 246,
+      "##li": 247,
+      "##em": 248,
+      "##ere": 249,
+      "##end": 250,
+      "##ip": 251,
+      "##ion": 252,
+      "##igh": 253,
+      "##ss": 254,
+      "##ses": 255,
+      "##te": 256,
+      "##tun": 257,
+      "##fe": 258,
+      "##wer": 259,
+      "##and": 260,
+      "##all": 261,
+      "##ant": 262,
+      "##ow": 263,
+      "##ore": 264,
       "##ue": 265,
       "##us": 266,
-      "##ck": 267,
-      "##cit": 268,
-      "##ver": 269,
-      "##vil": 270,
-      "##fe": 271,
+      "##ver": 267,
+      "##vil": 268,
+      "##ck": 269,
+      "##cit": 270,
+      "##ban": 271,
       "##ress": 272,
       "them": 273,
       "there": 274,
@@ -428,23 +428,23 @@
       "##ease": 280,
       "cont": 281,
       "comp": 282,
-      "##one": 283,
-      "##elf": 284,
+      "##elf": 283,
+      "##one": 284,
       "forth": 285,
       "canst": 286,
-      "calib": 287,
+      "cali": 287,
       "mere": 288,
       "ship": 289,
       "shall": 290,
-      "##own": 291,
+      "##lain": 291,
       "like": 292,
       "your": 293,
-      "##lete": 294,
-      "bad": 295,
-      "devil": 296,
-      "love": 297,
-      "seek": 298,
-      "work": 299
+      "good": 294,
+      "how": 295,
+      "love": 296,
+      "##lete": 297,
+      "bad": 298,
+      "devil": 299
     }
   }
 }

vocab.txt

@@ -32,30 +32,30 @@ w
 x
 y
 z
-##o
-##r
-##g
-##t
-##i
 ##l
 ##e
+##p
+##i
 ##s
+##t
+##r
+##f
+##w
 ##a
-##n
+##o
 ##u
-##c
+##n
+##d
+##g
 ##h
 ##v
-##y
 ##k
-##p
-##w
-##d
 ##m
+##z
+##y
+##c
 ##b
-##f
 ##x
-##z
 th
 ##ou
 ##re
@@ -68,8 +68,8 @@ my
 ##or
 ##ve
 ha
-##it
 ##ll
+##it
 to
 ##nt
 and
@@ -80,23 +80,23 @@ mo
 ##at
 in
 thou
-##in
 ##ea
+##in
 ##me
 co
 of
 ##ir
 wh
-##on
 ##el
+##on
 not
 is
 wi
 ##ee
 as
-##ra
-##th
 ##ld
+##th
+##ra
 most
 for
 ##ri
@@ -106,23 +106,23 @@ ca
 me
 so
 sh
-##ow
-##il
+##la
 ##en
 ##se
+##ro
 ##ch
 thy
 ##est
 have
 what
 are
+do
 it
 li
 sp
 you
-##oo
+##il
 ##ty
-##la
 ##ar
 ##ay
 ##ke
@@ -133,9 +133,10 @@ come
 ##irit
 spirit
 ch
-do
+go
+ho
 his
-##ro
+lo
 ##le
 ##ly
 ##ut
@@ -146,17 +147,17 @@ all
 ba
 but
 de
-lo
 st
 see
 we
 wor
 with
-##gh
 ##ic
+##wn
 ##an
 ##ake
 ##ul
+##gh
 ##mp
 ##ber
 that
@@ -168,9 +169,8 @@ now
 am
 ari
 ex
-go
-ho
 he
+po
 per
 re
 sou
@@ -179,42 +179,43 @@ sla
 say
 tis
 wa
-##ge
-##id
 ##et
 ##ement
-##ne
-##nst
+##id
+##for
+##ful
+##od
+##un
 ##ure
-##ct
+##nst
+##ge
 ##ves
 ##ms
 ##med
-##for
-##ful
-##our
+##ct
 ##ous
+##our
 they
 ##ish
 ##ess
 ##nter
 ##eas
+##rom
 liber
-##ood
 char
-##rom
 ariel
 serv
 slave
 liberty
 ad
-bo
 br
+bo
 by
 bes
 bra
 bear
 du
+en
 ear
 fre
 fir
@@ -224,52 +225,51 @@ fet
 from
 gre
 gra
-good
 hon
-how
 kn
 king
 make
 mist
 ne
-on
 ou
-ow
-po
+on
+own
 pl
 qu
 rel
 sl
+sw
 sa
 su
-sw
 sen
-tr
+vi
 would
 wish
-##ore
-##te
-##tu
+##li
+##em
+##ere
+##end
 ##ip
-##ib
 ##ion
 ##igh
-##lib
-##ere
-##end
 ##ss
 ##ses
+##te
+##tun
+##fe
+##wer
 ##and
 ##all
 ##ant
-##ain
+##ow
+##ore
 ##ue
 ##us
-##ck
-##cit
 ##ver
 ##vil
-##fe
+##ck
+##cit
+##ban
 ##ress
 them
 there
@@ -281,20 +281,20 @@ hath
 ##ease
 cont
 comp
-##one
 ##elf
+##one
 forth
 canst
-calib
+cali
 mere
 ship
 shall
-##own
+##lain
 like
 your
+good
+how
+love
 ##lete
 bad
 devil
-love
-seek
-work