Upload tokenizer

Browse files

Files changed (5) hide show

added_tokens.json +210 -0
special_tokens_map.json +221 -0
spiece.model +3 -0
tokenizer.json +0 -0
tokenizer_config.json +23 -0

added_tokens.json ADDED Viewed

	@@ -0,0 +1,210 @@

+{
+  "<pad>": 30002,
+  "[CLS]": 30000,
+  "[MASK]": 30003,
+  "[SEP]": 30001,
+  "[XXXXX0]": 30004,
+  "[XXXXX100]": 30104,
+  "[XXXXX101]": 30105,
+  "[XXXXX102]": 30106,
+  "[XXXXX103]": 30107,
+  "[XXXXX104]": 30108,
+  "[XXXXX105]": 30109,
+  "[XXXXX106]": 30110,
+  "[XXXXX107]": 30111,
+  "[XXXXX108]": 30112,
+  "[XXXXX109]": 30113,
+  "[XXXXX10]": 30014,
+  "[XXXXX110]": 30114,
+  "[XXXXX111]": 30115,
+  "[XXXXX112]": 30116,
+  "[XXXXX113]": 30117,
+  "[XXXXX114]": 30118,
+  "[XXXXX115]": 30119,
+  "[XXXXX116]": 30120,
+  "[XXXXX117]": 30121,
+  "[XXXXX118]": 30122,
+  "[XXXXX119]": 30123,
+  "[XXXXX11]": 30015,
+  "[XXXXX120]": 30124,
+  "[XXXXX121]": 30125,
+  "[XXXXX122]": 30126,
+  "[XXXXX123]": 30127,
+  "[XXXXX124]": 30128,
+  "[XXXXX125]": 30129,
+  "[XXXXX126]": 30130,
+  "[XXXXX127]": 30131,
+  "[XXXXX128]": 30132,
+  "[XXXXX129]": 30133,
+  "[XXXXX12]": 30016,
+  "[XXXXX130]": 30134,
+  "[XXXXX131]": 30135,
+  "[XXXXX132]": 30136,
+  "[XXXXX133]": 30137,
+  "[XXXXX134]": 30138,
+  "[XXXXX135]": 30139,
+  "[XXXXX136]": 30140,
+  "[XXXXX137]": 30141,
+  "[XXXXX138]": 30142,
+  "[XXXXX139]": 30143,
+  "[XXXXX13]": 30017,
+  "[XXXXX140]": 30144,
+  "[XXXXX141]": 30145,
+  "[XXXXX142]": 30146,
+  "[XXXXX143]": 30147,
+  "[XXXXX144]": 30148,
+  "[XXXXX145]": 30149,
+  "[XXXXX146]": 30150,
+  "[XXXXX147]": 30151,
+  "[XXXXX148]": 30152,
+  "[XXXXX149]": 30153,
+  "[XXXXX14]": 30018,
+  "[XXXXX150]": 30154,
+  "[XXXXX151]": 30155,
+  "[XXXXX152]": 30156,
+  "[XXXXX153]": 30157,
+  "[XXXXX154]": 30158,
+  "[XXXXX155]": 30159,
+  "[XXXXX156]": 30160,
+  "[XXXXX157]": 30161,
+  "[XXXXX158]": 30162,
+  "[XXXXX159]": 30163,
+  "[XXXXX15]": 30019,
+  "[XXXXX160]": 30164,
+  "[XXXXX161]": 30165,
+  "[XXXXX162]": 30166,
+  "[XXXXX163]": 30167,
+  "[XXXXX164]": 30168,
+  "[XXXXX165]": 30169,
+  "[XXXXX166]": 30170,
+  "[XXXXX167]": 30171,
+  "[XXXXX168]": 30172,
+  "[XXXXX169]": 30173,
+  "[XXXXX16]": 30020,
+  "[XXXXX170]": 30174,
+  "[XXXXX171]": 30175,
+  "[XXXXX172]": 30176,
+  "[XXXXX173]": 30177,
+  "[XXXXX174]": 30178,
+  "[XXXXX175]": 30179,
+  "[XXXXX176]": 30180,
+  "[XXXXX177]": 30181,
+  "[XXXXX178]": 30182,
+  "[XXXXX179]": 30183,
+  "[XXXXX17]": 30021,
+  "[XXXXX180]": 30184,
+  "[XXXXX181]": 30185,
+  "[XXXXX182]": 30186,
+  "[XXXXX183]": 30187,
+  "[XXXXX184]": 30188,
+  "[XXXXX185]": 30189,
+  "[XXXXX186]": 30190,
+  "[XXXXX187]": 30191,
+  "[XXXXX188]": 30192,
+  "[XXXXX189]": 30193,
+  "[XXXXX18]": 30022,
+  "[XXXXX190]": 30194,
+  "[XXXXX191]": 30195,
+  "[XXXXX192]": 30196,
+  "[XXXXX193]": 30197,
+  "[XXXXX194]": 30198,
+  "[XXXXX195]": 30199,
+  "[XXXXX196]": 30200,
+  "[XXXXX197]": 30201,
+  "[XXXXX198]": 30202,
+  "[XXXXX199]": 30203,
+  "[XXXXX19]": 30023,
+  "[XXXXX1]": 30005,
+  "[XXXXX200]": 30204,
+  "[XXXXX201]": 30205,
+  "[XXXXX202]": 30206,
+  "[XXXXX203]": 30207,
+  "[XXXXX20]": 30024,
+  "[XXXXX21]": 30025,
+  "[XXXXX22]": 30026,
+  "[XXXXX23]": 30027,
+  "[XXXXX24]": 30028,
+  "[XXXXX25]": 30029,
+  "[XXXXX26]": 30030,
+  "[XXXXX27]": 30031,
+  "[XXXXX28]": 30032,
+  "[XXXXX29]": 30033,
+  "[XXXXX2]": 30006,
+  "[XXXXX30]": 30034,
+  "[XXXXX31]": 30035,
+  "[XXXXX32]": 30036,
+  "[XXXXX33]": 30037,
+  "[XXXXX34]": 30038,
+  "[XXXXX35]": 30039,
+  "[XXXXX36]": 30040,
+  "[XXXXX37]": 30041,
+  "[XXXXX38]": 30042,
+  "[XXXXX39]": 30043,
+  "[XXXXX3]": 30007,
+  "[XXXXX40]": 30044,
+  "[XXXXX41]": 30045,
+  "[XXXXX42]": 30046,
+  "[XXXXX43]": 30047,
+  "[XXXXX44]": 30048,
+  "[XXXXX45]": 30049,
+  "[XXXXX46]": 30050,
+  "[XXXXX47]": 30051,
+  "[XXXXX48]": 30052,
+  "[XXXXX49]": 30053,
+  "[XXXXX4]": 30008,
+  "[XXXXX50]": 30054,
+  "[XXXXX51]": 30055,
+  "[XXXXX52]": 30056,
+  "[XXXXX53]": 30057,
+  "[XXXXX54]": 30058,
+  "[XXXXX55]": 30059,
+  "[XXXXX56]": 30060,
+  "[XXXXX57]": 30061,
+  "[XXXXX58]": 30062,
+  "[XXXXX59]": 30063,
+  "[XXXXX5]": 30009,
+  "[XXXXX60]": 30064,
+  "[XXXXX61]": 30065,
+  "[XXXXX62]": 30066,
+  "[XXXXX63]": 30067,
+  "[XXXXX64]": 30068,
+  "[XXXXX65]": 30069,
+  "[XXXXX66]": 30070,
+  "[XXXXX67]": 30071,
+  "[XXXXX68]": 30072,
+  "[XXXXX69]": 30073,
+  "[XXXXX6]": 30010,
+  "[XXXXX70]": 30074,
+  "[XXXXX71]": 30075,
+  "[XXXXX72]": 30076,
+  "[XXXXX73]": 30077,
+  "[XXXXX74]": 30078,
+  "[XXXXX75]": 30079,
+  "[XXXXX76]": 30080,
+  "[XXXXX77]": 30081,
+  "[XXXXX78]": 30082,
+  "[XXXXX79]": 30083,
+  "[XXXXX7]": 30011,
+  "[XXXXX80]": 30084,
+  "[XXXXX81]": 30085,
+  "[XXXXX82]": 30086,
+  "[XXXXX83]": 30087,
+  "[XXXXX84]": 30088,
+  "[XXXXX85]": 30089,
+  "[XXXXX86]": 30090,
+  "[XXXXX87]": 30091,
+  "[XXXXX88]": 30092,
+  "[XXXXX89]": 30093,
+  "[XXXXX8]": 30012,
+  "[XXXXX90]": 30094,
+  "[XXXXX91]": 30095,
+  "[XXXXX92]": 30096,
+  "[XXXXX93]": 30097,
+  "[XXXXX94]": 30098,
+  "[XXXXX95]": 30099,
+  "[XXXXX96]": 30100,
+  "[XXXXX97]": 30101,
+  "[XXXXX98]": 30102,
+  "[XXXXX99]": 30103,
+  "[XXXXX9]": 30013
+}

special_tokens_map.json ADDED Viewed

	@@ -0,0 +1,221 @@

+{
+  "additional_special_tokens": [
+    "[XXXXX0]",
+    "[XXXXX1]",
+    "[XXXXX2]",
+    "[XXXXX3]",
+    "[XXXXX4]",
+    "[XXXXX5]",
+    "[XXXXX6]",
+    "[XXXXX7]",
+    "[XXXXX8]",
+    "[XXXXX9]",
+    "[XXXXX10]",
+    "[XXXXX11]",
+    "[XXXXX12]",
+    "[XXXXX13]",
+    "[XXXXX14]",
+    "[XXXXX15]",
+    "[XXXXX16]",
+    "[XXXXX17]",
+    "[XXXXX18]",
+    "[XXXXX19]",
+    "[XXXXX20]",
+    "[XXXXX21]",
+    "[XXXXX22]",
+    "[XXXXX23]",
+    "[XXXXX24]",
+    "[XXXXX25]",
+    "[XXXXX26]",
+    "[XXXXX27]",
+    "[XXXXX28]",
+    "[XXXXX29]",
+    "[XXXXX30]",
+    "[XXXXX31]",
+    "[XXXXX32]",
+    "[XXXXX33]",
+    "[XXXXX34]",
+    "[XXXXX35]",
+    "[XXXXX36]",
+    "[XXXXX37]",
+    "[XXXXX38]",
+    "[XXXXX39]",
+    "[XXXXX40]",
+    "[XXXXX41]",
+    "[XXXXX42]",
+    "[XXXXX43]",
+    "[XXXXX44]",
+    "[XXXXX45]",
+    "[XXXXX46]",
+    "[XXXXX47]",
+    "[XXXXX48]",
+    "[XXXXX49]",
+    "[XXXXX50]",
+    "[XXXXX51]",
+    "[XXXXX52]",
+    "[XXXXX53]",
+    "[XXXXX54]",
+    "[XXXXX55]",
+    "[XXXXX56]",
+    "[XXXXX57]",
+    "[XXXXX58]",
+    "[XXXXX59]",
+    "[XXXXX60]",
+    "[XXXXX61]",
+    "[XXXXX62]",
+    "[XXXXX63]",
+    "[XXXXX64]",
+    "[XXXXX65]",
+    "[XXXXX66]",
+    "[XXXXX67]",
+    "[XXXXX68]",
+    "[XXXXX69]",
+    "[XXXXX70]",
+    "[XXXXX71]",
+    "[XXXXX72]",
+    "[XXXXX73]",
+    "[XXXXX74]",
+    "[XXXXX75]",
+    "[XXXXX76]",
+    "[XXXXX77]",
+    "[XXXXX78]",
+    "[XXXXX79]",
+    "[XXXXX80]",
+    "[XXXXX81]",
+    "[XXXXX82]",
+    "[XXXXX83]",
+    "[XXXXX84]",
+    "[XXXXX85]",
+    "[XXXXX86]",
+    "[XXXXX87]",
+    "[XXXXX88]",
+    "[XXXXX89]",
+    "[XXXXX90]",
+    "[XXXXX91]",
+    "[XXXXX92]",
+    "[XXXXX93]",
+    "[XXXXX94]",
+    "[XXXXX95]",
+    "[XXXXX96]",
+    "[XXXXX97]",
+    "[XXXXX98]",
+    "[XXXXX99]",
+    "[XXXXX100]",
+    "[XXXXX101]",
+    "[XXXXX102]",
+    "[XXXXX103]",
+    "[XXXXX104]",
+    "[XXXXX105]",
+    "[XXXXX106]",
+    "[XXXXX107]",
+    "[XXXXX108]",
+    "[XXXXX109]",
+    "[XXXXX110]",
+    "[XXXXX111]",
+    "[XXXXX112]",
+    "[XXXXX113]",
+    "[XXXXX114]",
+    "[XXXXX115]",
+    "[XXXXX116]",
+    "[XXXXX117]",
+    "[XXXXX118]",
+    "[XXXXX119]",
+    "[XXXXX120]",
+    "[XXXXX121]",
+    "[XXXXX122]",
+    "[XXXXX123]",
+    "[XXXXX124]",
+    "[XXXXX125]",
+    "[XXXXX126]",
+    "[XXXXX127]",
+    "[XXXXX128]",
+    "[XXXXX129]",
+    "[XXXXX130]",
+    "[XXXXX131]",
+    "[XXXXX132]",
+    "[XXXXX133]",
+    "[XXXXX134]",
+    "[XXXXX135]",
+    "[XXXXX136]",
+    "[XXXXX137]",
+    "[XXXXX138]",
+    "[XXXXX139]",
+    "[XXXXX140]",
+    "[XXXXX141]",
+    "[XXXXX142]",
+    "[XXXXX143]",
+    "[XXXXX144]",
+    "[XXXXX145]",
+    "[XXXXX146]",
+    "[XXXXX147]",
+    "[XXXXX148]",
+    "[XXXXX149]",
+    "[XXXXX150]",
+    "[XXXXX151]",
+    "[XXXXX152]",
+    "[XXXXX153]",
+    "[XXXXX154]",
+    "[XXXXX155]",
+    "[XXXXX156]",
+    "[XXXXX157]",
+    "[XXXXX158]",
+    "[XXXXX159]",
+    "[XXXXX160]",
+    "[XXXXX161]",
+    "[XXXXX162]",
+    "[XXXXX163]",
+    "[XXXXX164]",
+    "[XXXXX165]",
+    "[XXXXX166]",
+    "[XXXXX167]",
+    "[XXXXX168]",
+    "[XXXXX169]",
+    "[XXXXX170]",
+    "[XXXXX171]",
+    "[XXXXX172]",
+    "[XXXXX173]",
+    "[XXXXX174]",
+    "[XXXXX175]",
+    "[XXXXX176]",
+    "[XXXXX177]",
+    "[XXXXX178]",
+    "[XXXXX179]",
+    "[XXXXX180]",
+    "[XXXXX181]",
+    "[XXXXX182]",
+    "[XXXXX183]",
+    "[XXXXX184]",
+    "[XXXXX185]",
+    "[XXXXX186]",
+    "[XXXXX187]",
+    "[XXXXX188]",
+    "[XXXXX189]",
+    "[XXXXX190]",
+    "[XXXXX191]",
+    "[XXXXX192]",
+    "[XXXXX193]",
+    "[XXXXX194]",
+    "[XXXXX195]",
+    "[XXXXX196]",
+    "[XXXXX197]",
+    "[XXXXX198]",
+    "[XXXXX199]",
+    "[XXXXX200]",
+    "[XXXXX201]",
+    "[XXXXX202]",
+    "[XXXXX203]"
+  ],
+  "bos_token": "[CLS]",
+  "cls_token": "[CLS]",
+  "eos_token": "[SEP]",
+  "mask_token": {
+    "content": "[MASK]",
+    "lstrip": true,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<pad>",
+  "sep_token": "[SEP]",
+  "unk_token": "<unk>"
+}

spiece.model ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:1840794a944b1ca5d804431a8fb26e42fa55680e0c845315adef099c24d322d5
+size 756326

tokenizer.json ADDED Viewed

The diff for this file is too large to render. See raw diff

tokenizer_config.json ADDED Viewed

	@@ -0,0 +1,23 @@

+{
+  "bos_token": "[CLS]",
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "[CLS]",
+  "do_lower_case": false,
+  "eos_token": "[SEP]",
+  "keep_accents": false,
+  "mask_token": {
+    "__type": "AddedToken",
+    "content": "[MASK]",
+    "lstrip": true,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "remove_space": true,
+  "sep_token": "[SEP]",
+  "sp_model_kwargs": {},
+  "tokenizer_class": "AlbertTokenizer",
+  "unk_token": "<unk>"
+}