Upload tokenizer

Browse files

Files changed (6) hide show

added_tokens.json +206 -104
merges.txt +0 -0
special_tokens_map.json +0 -1
tokenizer.json +0 -0
tokenizer_config.json +1 -1
vocab.json +0 -0

added_tokens.json CHANGED Viewed

@@ -1,106 +1,208 @@
 {
-  "[ex0]": 31972,
-  "[ex10]": 31982,
-  "[ex11]": 31983,
-  "[ex12]": 31984,
-  "[ex13]": 31985,
-  "[ex14]": 31986,
-  "[ex15]": 31987,
-  "[ex16]": 31988,
-  "[ex17]": 31989,
-  "[ex18]": 31990,
-  "[ex19]": 31991,
-  "[ex1]": 31973,
-  "[ex20]": 31992,
-  "[ex21]": 31993,
-  "[ex22]": 31994,
-  "[ex23]": 31995,
-  "[ex24]": 31996,
-  "[ex25]": 31997,
-  "[ex26]": 31998,
-  "[ex27]": 31999,
-  "[ex28]": 32000,
-  "[ex29]": 32001,
-  "[ex2]": 31974,
-  "[ex30]": 32002,
-  "[ex31]": 32003,
-  "[ex32]": 32004,
-  "[ex33]": 32005,
-  "[ex34]": 32006,
-  "[ex35]": 32007,
-  "[ex36]": 32008,
-  "[ex37]": 32009,
-  "[ex38]": 32010,
-  "[ex39]": 32011,
-  "[ex3]": 31975,
-  "[ex40]": 32012,
-  "[ex41]": 32013,
-  "[ex42]": 32014,
-  "[ex43]": 32015,
-  "[ex44]": 32016,
-  "[ex45]": 32017,
-  "[ex46]": 32018,
-  "[ex47]": 32019,
-  "[ex48]": 32020,
-  "[ex49]": 32021,
-  "[ex4]": 31976,
-  "[ex50]": 32022,
-  "[ex51]": 32023,
-  "[ex52]": 32024,
-  "[ex53]": 32025,
-  "[ex54]": 32026,
-  "[ex55]": 32027,
-  "[ex56]": 32028,
-  "[ex57]": 32029,
-  "[ex58]": 32030,
-  "[ex59]": 32031,
-  "[ex5]": 31977,
-  "[ex60]": 32032,
-  "[ex61]": 32033,
-  "[ex62]": 32034,
-  "[ex63]": 32035,
-  "[ex64]": 32036,
-  "[ex65]": 32037,
-  "[ex66]": 32038,
-  "[ex67]": 32039,
-  "[ex68]": 32040,
-  "[ex69]": 32041,
-  "[ex6]": 31978,
-  "[ex70]": 32042,
-  "[ex71]": 32043,
-  "[ex72]": 32044,
-  "[ex73]": 32045,
-  "[ex74]": 32046,
-  "[ex75]": 32047,
-  "[ex76]": 32048,
-  "[ex77]": 32049,
-  "[ex78]": 32050,
-  "[ex79]": 32051,
-  "[ex7]": 31979,
-  "[ex80]": 32052,
-  "[ex81]": 32053,
-  "[ex82]": 32054,
-  "[ex83]": 32055,
-  "[ex84]": 32056,
-  "[ex85]": 32057,
-  "[ex86]": 32058,
-  "[ex87]": 32059,
-  "[ex88]": 32060,
-  "[ex89]": 32061,
-  "[ex8]": 31980,
-  "[ex90]": 32062,
-  "[ex91]": 32063,
-  "[ex92]": 32064,
-  "[ex93]": 32065,
-  "[ex94]": 32066,
-  "[ex95]": 32067,
-  "[ex96]": 32068,
-  "[ex97]": 32069,
-  "[ex98]": 32070,
-  "[ex99]": 32071,
-  "[ex9]": 31981,
-  "[frl]": 31970,
-  "[mrl]": 31971,
-  "ext.": 32072,
-  "int.": 32073
 }

 {
+  "EXT.": 41061,
+  "INT.": 41062,
+  "[EX0]": 41065,
+  "[EX0]:": 41167,
+  "[EX10]": 41075,
+  "[EX10]:": 41177,
+  "[EX11]": 41076,
+  "[EX11]:": 41178,
+  "[EX12]": 41077,
+  "[EX12]:": 41179,
+  "[EX13]": 41078,
+  "[EX13]:": 41180,
+  "[EX14]": 41079,
+  "[EX14]:": 41181,
+  "[EX15]": 41080,
+  "[EX15]:": 41182,
+  "[EX16]": 41081,
+  "[EX16]:": 41183,
+  "[EX17]": 41082,
+  "[EX17]:": 41184,
+  "[EX18]": 41083,
+  "[EX18]:": 41185,
+  "[EX19]": 41084,
+  "[EX19]:": 41186,
+  "[EX1]": 41066,
+  "[EX1]:": 41168,
+  "[EX20]": 41085,
+  "[EX20]:": 41187,
+  "[EX21]": 41086,
+  "[EX21]:": 41188,
+  "[EX22]": 41087,
+  "[EX22]:": 41189,
+  "[EX23]": 41088,
+  "[EX23]:": 41190,
+  "[EX24]": 41089,
+  "[EX24]:": 41191,
+  "[EX25]": 41090,
+  "[EX25]:": 41192,
+  "[EX26]": 41091,
+  "[EX26]:": 41193,
+  "[EX27]": 41092,
+  "[EX27]:": 41194,
+  "[EX28]": 41093,
+  "[EX28]:": 41195,
+  "[EX29]": 41094,
+  "[EX29]:": 41196,
+  "[EX2]": 41067,
+  "[EX2]:": 41169,
+  "[EX30]": 41095,
+  "[EX30]:": 41197,
+  "[EX31]": 41096,
+  "[EX31]:": 41198,
+  "[EX32]": 41097,
+  "[EX32]:": 41199,
+  "[EX33]": 41098,
+  "[EX33]:": 41200,
+  "[EX34]": 41099,
+  "[EX34]:": 41201,
+  "[EX35]": 41100,
+  "[EX35]:": 41202,
+  "[EX36]": 41101,
+  "[EX36]:": 41203,
+  "[EX37]": 41102,
+  "[EX37]:": 41204,
+  "[EX38]": 41103,
+  "[EX38]:": 41205,
+  "[EX39]": 41104,
+  "[EX39]:": 41206,
+  "[EX3]": 41068,
+  "[EX3]:": 41170,
+  "[EX40]": 41105,
+  "[EX40]:": 41207,
+  "[EX41]": 41106,
+  "[EX41]:": 41208,
+  "[EX42]": 41107,
+  "[EX42]:": 41209,
+  "[EX43]": 41108,
+  "[EX43]:": 41210,
+  "[EX44]": 41109,
+  "[EX44]:": 41211,
+  "[EX45]": 41110,
+  "[EX45]:": 41212,
+  "[EX46]": 41111,
+  "[EX46]:": 41213,
+  "[EX47]": 41112,
+  "[EX47]:": 41214,
+  "[EX48]": 41113,
+  "[EX48]:": 41215,
+  "[EX49]": 41114,
+  "[EX49]:": 41216,
+  "[EX4]": 41069,
+  "[EX4]:": 41171,
+  "[EX50]": 41115,
+  "[EX50]:": 41217,
+  "[EX51]": 41116,
+  "[EX51]:": 41218,
+  "[EX52]": 41117,
+  "[EX52]:": 41219,
+  "[EX53]": 41118,
+  "[EX53]:": 41220,
+  "[EX54]": 41119,
+  "[EX54]:": 41221,
+  "[EX55]": 41120,
+  "[EX55]:": 41222,
+  "[EX56]": 41121,
+  "[EX56]:": 41223,
+  "[EX57]": 41122,
+  "[EX57]:": 41224,
+  "[EX58]": 41123,
+  "[EX58]:": 41225,
+  "[EX59]": 41124,
+  "[EX59]:": 41226,
+  "[EX5]": 41070,
+  "[EX5]:": 41172,
+  "[EX60]": 41125,
+  "[EX60]:": 41227,
+  "[EX61]": 41126,
+  "[EX61]:": 41228,
+  "[EX62]": 41127,
+  "[EX62]:": 41229,
+  "[EX63]": 41128,
+  "[EX63]:": 41230,
+  "[EX64]": 41129,
+  "[EX64]:": 41231,
+  "[EX65]": 41130,
+  "[EX65]:": 41232,
+  "[EX66]": 41131,
+  "[EX66]:": 41233,
+  "[EX67]": 41132,
+  "[EX67]:": 41234,
+  "[EX68]": 41133,
+  "[EX68]:": 41235,
+  "[EX69]": 41134,
+  "[EX69]:": 41236,
+  "[EX6]": 41071,
+  "[EX6]:": 41173,
+  "[EX70]": 41135,
+  "[EX70]:": 41237,
+  "[EX71]": 41136,
+  "[EX71]:": 41238,
+  "[EX72]": 41137,
+  "[EX72]:": 41239,
+  "[EX73]": 41138,
+  "[EX73]:": 41240,
+  "[EX74]": 41139,
+  "[EX74]:": 41241,
+  "[EX75]": 41140,
+  "[EX75]:": 41242,
+  "[EX76]": 41141,
+  "[EX76]:": 41243,
+  "[EX77]": 41142,
+  "[EX77]:": 41244,
+  "[EX78]": 41143,
+  "[EX78]:": 41245,
+  "[EX79]": 41144,
+  "[EX79]:": 41246,
+  "[EX7]": 41072,
+  "[EX7]:": 41174,
+  "[EX80]": 41145,
+  "[EX80]:": 41247,
+  "[EX81]": 41146,
+  "[EX81]:": 41248,
+  "[EX82]": 41147,
+  "[EX82]:": 41249,
+  "[EX83]": 41148,
+  "[EX83]:": 41250,
+  "[EX84]": 41149,
+  "[EX84]:": 41251,
+  "[EX85]": 41150,
+  "[EX85]:": 41252,
+  "[EX86]": 41151,
+  "[EX86]:": 41253,
+  "[EX87]": 41152,
+  "[EX87]:": 41254,
+  "[EX88]": 41153,
+  "[EX88]:": 41255,
+  "[EX89]": 41154,
+  "[EX89]:": 41256,
+  "[EX8]": 41073,
+  "[EX8]:": 41175,
+  "[EX90]": 41155,
+  "[EX90]:": 41257,
+  "[EX91]": 41156,
+  "[EX91]:": 41258,
+  "[EX92]": 41157,
+  "[EX92]:": 41259,
+  "[EX93]": 41158,
+  "[EX93]:": 41260,
+  "[EX94]": 41159,
+  "[EX94]:": 41261,
+  "[EX95]": 41160,
+  "[EX95]:": 41262,
+  "[EX96]": 41161,
+  "[EX96]:": 41263,
+  "[EX97]": 41162,
+  "[EX97]:": 41264,
+  "[EX98]": 41163,
+  "[EX98]:": 41265,
+  "[EX99]": 41164,
+  "[EX99]:": 41266,
+  "[EX9]": 41074,
+  "[EX9]:": 41176,
+  "[FRL]": 41063,
+  "[FRL]:": 41165,
+  "[MRL]": 41064,
+  "[MRL]:": 41166
 }

merges.txt CHANGED Viewed

The diff for this file is too large to render. See raw diff

special_tokens_map.json CHANGED Viewed

@@ -1,6 +1,5 @@
 {
   "bos_token": "<|endoftext|>",
   "eos_token": "<|endoftext|>",
-  "pad_token": "<|endoftext|>",
   "unk_token": "<|endoftext|>"
 }

 {
   "bos_token": "<|endoftext|>",
   "eos_token": "<|endoftext|>",
   "unk_token": "<|endoftext|>"
 }

tokenizer.json CHANGED Viewed

The diff for this file is too large to render. See raw diff

tokenizer_config.json CHANGED Viewed

@@ -3,7 +3,7 @@
   "bos_token": "<|endoftext|>",
   "eos_token": "<|endoftext|>",
   "model_max_length": 1024,
-  "name_or_path": "MarkGG/Romance-cleaned-3",
   "special_tokens_map_file": null,
   "tokenizer_class": "GPT2Tokenizer",
   "unk_token": "<|endoftext|>"

   "bos_token": "<|endoftext|>",
   "eos_token": "<|endoftext|>",
   "model_max_length": 1024,
+  "name_or_path": "gpt2",
   "special_tokens_map_file": null,
   "tokenizer_class": "GPT2Tokenizer",
   "unk_token": "<|endoftext|>"

vocab.json CHANGED Viewed

The diff for this file is too large to render. See raw diff