maykcaldas
/

selfies-bart

Transformers

JAX

bert

pretraining

Model card Files Files and versions

xet

Community

maykcaldas commited on Oct 19, 2022

Commit

5b69007

1 Parent(s): c8310f2

add tokenizer

Browse files

Files changed (2) hide show

tokenizer.json +278 -136
tokenizer_config.json +1 -3

tokenizer.json CHANGED Viewed

@@ -2,53 +2,7 @@
   "version": "1.0",
   "truncation": null,
   "padding": null,
-  "added_tokens": [
-    {
-      "id": 10,
-      "content": "[bos]",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 32,
-      "content": "[eos]",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 57,
-      "content": "[unk]",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 65,
-      "content": "[nop]",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 88,
-      "content": "[mask]",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    }
-  ],
   "normalizer": null,
   "pre_tokenizer": {
     "type": "WhitespaceSplit"
@@ -61,95 +15,283 @@
     "continuing_subword_prefix": "##",
     "max_input_chars_per_word": 100,
     "vocab": {
-      "[Branch1]": 0,
-      "[=NH1+1]": 1,
-      "[Se]": 2,
-      "[#Branch2]": 3,
-      "[O-1]": 4,
-      "[SiH1]": 5,
-      "[SeH1]": 6,
-      "[CH2-1]": 7,
-      "[SH0]": 8,
-      "[PH1]": 9,
-      "[bos]": 10,
-      "[Si]": 11,
-      "[OH1+1]": 12,
-      "[Fe]": 13,
-      "[NH1]": 14,
-      "[Ring2]": 15,
-      "[=N]": 16,
-      "[=NH2+1]": 17,
-      "[B]": 18,
-      "[=SH1]": 19,
-      "[C]": 20,
-      "[=C]": 21,
-      "[NH1-1]": 22,
-      "[=O+1]": 23,
-      "[As]": 24,
-      "[#Branch1]": 25,
-      "[I]": 26,
-      "[=O]": 27,
-      "[B-1]": 28,
-      "[Fe-4]": 29,
-      "[=Ring1]": 30,
-      "[=S]": 31,
-      "[eos]": 32,
-      "[Cl]": 33,
-      "[=P]": 34,
-      "[=Fe]": 35,
-      "[NH1+1]": 36,
-      "[CH1]": 37,
-      "[#Ring1]": 38,
-      "[As+1]": 39,
-      "[Branch3]": 40,
-      "[O]": 41,
-      "[=OH1+1]": 42,
-      "[Branch2]": 43,
-      "[=As]": 44,
-      "[F]": 45,
-      "[P+1]": 46,
-      "[S]": 47,
-      "[#Ring2]": 48,
-      "[#N]": 49,
-      "[CH1+1]": 50,
-      "[OH0]": 51,
-      "[N]": 52,
-      "[I+1]": 53,
-      "[=Ring2]": 54,
-      "[C+1]": 55,
-      "[=B]": 56,
-      "[unk]": 57,
-      "[SiH2]": 58,
-      "[C-1]": 59,
-      "[=PH1]": 60,
-      "[#C]": 61,
-      "[SH1]": 62,
-      "[Fe-3]": 63,
-      "[Br]": 64,
-      "[nop]": 65,
-      "[CH1-1]": 66,
-      "[NH3+1]": 67,
-      "[=Branch1]": 68,
-      "[NH2+1]": 69,
-      "[P]": 70,
-      "[K]": 71,
-      "[N+1]": 72,
-      "[CH0]": 73,
-      "[=Se]": 74,
-      "[Fe+1]": 75,
-      "[Ring1]": 76,
-      "[S+1]": 77,
-      "[=Branch3]": 78,
-      "[Fe+2]": 79,
-      "[=S+1]": 80,
-      "[=N+1]": 81,
-      "[Na]": 82,
-      "[Se+1]": 83,
-      "[N-1]": 84,
-      "[NH0]": 85,
-      "[#S]": 86,
-      "[=Branch2]": 87,
-      "[mask]": 88
     }
   }
 }

   "version": "1.0",
   "truncation": null,
   "padding": null,
+  "added_tokens": [],
   "normalizer": null,
   "pre_tokenizer": {
     "type": "WhitespaceSplit"
     "continuing_subword_prefix": "##",
     "max_input_chars_per_word": 100,
     "vocab": {
+      "[mask]": 0,
+      "[bos]": 1,
+      "[eos]": 2,
+      "[unk]": 3,
+      "[nop]": 4,
+      ".": 5,
+      "[N@]": 6,
+      "[Se]": 7,
+      "[Li+1]": 8,
+      "[=Ge]": 9,
+      "[\\N]": 10,
+      "[Mg+2]": 11,
+      "[ClH0]": 12,
+      "[Ba+2]": 13,
+      "[N]": 14,
+      "[=Mo]": 15,
+      "[SH1-2]": 16,
+      "[=O]": 17,
+      "[Zr+4]": 18,
+      "[\\N+1]": 19,
+      "[/Br]": 20,
+      "[Mo+2]": 21,
+      "[/-Ring1]": 22,
+      "[=SH0]": 23,
+      "[Sn]": 24,
+      "[=Si]": 25,
+      "[N-1]": 26,
+      "[Fe+2]": 27,
+      "[FH0]": 28,
+      "[B-1]": 29,
+      "[2H]": 30,
+      "[/N+1]": 31,
+      "[Cl-1]": 32,
+      "[=V+2]": 33,
+      "[\\NH1+1]": 34,
+      "[=N]": 35,
+      "[CH0]": 36,
+      "[P@@]": 37,
+      "[=Fe]": 38,
+      "[\\S]": 39,
+      "[Cl]": 40,
+      "[Al+3]": 41,
+      "[\\-Ring2]": 42,
+      "[As+3]": 43,
+      "[Ring1]": 44,
+      "[Sm+3]": 45,
+      "[=Ni]": 46,
+      "[Co+2]": 47,
+      "[NH1+2]": 48,
+      "[\\C@H1]": 49,
+      "[Mo]": 50,
+      "[S-2]": 51,
+      "[=PH0]": 52,
+      "[Zn]": 53,
+      "[Cr]": 54,
+      "[Br-1]": 55,
+      "[=P]": 56,
+      "[S]": 57,
+      "[#Branch2]": 58,
+      "[-/Ring1]": 59,
+      "[\\C@@H1]": 60,
+      "[NH2+1]": 61,
+      "[Sr]": 62,
+      "[=Cr]": 63,
+      "[=N-1]": 64,
+      "[=SnH2]": 65,
+      "[\\NH1]": 66,
+      "[PH0]": 67,
+      "[NH0]": 68,
+      "[Gd]": 69,
+      "[Na]": 70,
+      "[/C@@H1]": 71,
+      "[Au+1]": 72,
+      "[Ce]": 73,
+      "[Dy]": 74,
+      "[=Ti]": 75,
+      "[CH2]": 76,
+      "[Hg]": 77,
+      "[=Ring2]": 78,
+      "[=Zr]": 79,
+      "[Zr+2]": 80,
+      "[I]": 81,
+      "[Si]": 82,
+      "[=Branch1]": 83,
+      "[K]": 84,
+      "[BH0]": 85,
+      "[Pr+3]": 86,
+      "[Nd+3]": 87,
+      "[Ag+1]": 88,
+      "[In+3]": 89,
+      "[Cl+3]": 90,
+      "[Cu]": 91,
+      "[O-2]": 92,
+      "[W]": 93,
+      "[N+1]": 94,
+      "[=Ca]": 95,
+      "[=S]": 96,
+      "[=P+1]": 97,
+      "[/N]": 98,
+      "[#O+1]": 99,
+      "[H+1]": 100,
+      "[Nb+5]": 101,
+      "[NH3+1]": 102,
+      "[Branch3]": 103,
+      "[Gd+3]": 104,
+      "[Zn+2]": 105,
+      "[/C@]": 106,
+      "[Cd+2]": 107,
+      "[Ce+3]": 108,
+      "[Pb+2]": 109,
+      "[Na+1]": 110,
+      "[#Branch1]": 111,
+      "[Lu+3]": 112,
+      "[MgH2]": 113,
+      "[\\Br]": 114,
+      "[-/Ring2]": 115,
+      "[NH1+1]": 116,
+      "[S-1]": 117,
+      "[I-1]": 118,
+      "[O]": 119,
+      "[=V]": 120,
+      "[Y]": 121,
+      "[Rh]": 122,
+      "[#S]": 123,
+      "[Cs+1]": 124,
+      "[Ni]": 125,
+      "[H]": 126,
+      "[#N]": 127,
+      "[Cu+1]": 128,
+      "[=PH1]": 129,
+      "[SH1-1]": 130,
+      "[=Mn]": 131,
+      "[Mn+3]": 132,
+      "[#C]": 133,
+      "[Branch2]": 134,
+      "[Sb+3]": 135,
+      "[=B]": 136,
+      "[\\As]": 137,
+      "[=Sn]": 138,
+      "[H-1]": 139,
+      "[=Ce]": 140,
+      "[B+3]": 141,
+      "[=O+1]": 142,
+      "[=Branch2]": 143,
+      "[K+1]": 144,
+      "[=Pb]": 145,
+      "[P@]": 146,
+      "[Al]": 147,
+      "[=Ring1]": 148,
+      "[Co]": 149,
+      "[Be]": 150,
+      "[=Se]": 151,
+      "[/I]": 152,
+      "[C@@H1]": 153,
+      "[/S]": 154,
+      "[Ring2]": 155,
+      "[Ni+2]": 156,
+      "[/As]": 157,
+      "[C]": 158,
+      "[Ta]": 159,
+      "[F-1]": 160,
+      "[Sb]": 161,
+      "[Au]": 162,
+      "[/NH1+1]": 163,
+      "[Zr]": 164,
+      "[CH1-1]": 165,
+      "[Rh+3]": 166,
+      "[Si@]": 167,
+      "[\\O]": 168,
+      "[-\\Ring1]": 169,
+      "[SnH2]": 170,
+      "[Bi+3]": 171,
+      "[Fe]": 172,
+      "[Hf]": 173,
+      "[=NH1+1]": 174,
+      "[Pb]": 175,
+      "[\\Cl]": 176,
+      "[Ca+2]": 177,
+      "[Ir]": 178,
+      "[Pt+2]": 179,
+      "[Hg+2]": 180,
+      "[#B]": 181,
+      "[Ge]": 182,
+      "[=Te]": 183,
+      "[Nd]": 184,
+      "[OH1-1]": 185,
+      "[NH1]": 186,
+      "[C+1]": 187,
+      "[Co+3]": 188,
+      "[=Cd]": 189,
+      "[Y+3]": 190,
+      "[Ti+2]": 191,
+      "[Au+3]": 192,
+      "[OH0]": 193,
+      "[Pd+2]": 194,
+      "[O+1]": 195,
+      "[=As]": 196,
+      "[/C]": 197,
+      "[Hg+1]": 198,
+      "[=N+1]": 199,
+      "[Sn+2]": 200,
+      "[O-1]": 201,
+      "[F]": 202,
+      "[NH4+1]": 203,
+      "[Re]": 204,
+      "[/O]": 205,
+      "[Sr+2]": 206,
+      "[/-Ring2]": 207,
+      "[Dy+3]": 208,
+      "[S+1]": 209,
+      "[Sn+4]": 210,
+      "[\\I]": 211,
+      "[Ti+4]": 212,
+      "[Mn+2]": 213,
+      "[Mg]": 214,
+      "[B]": 215,
+      "[La]": 216,
+      "[=W]": 217,
+      "[Cs]": 218,
+      "[#C-1]": 219,
+      "[=Sr]": 220,
+      "[=S+1]": 221,
+      "[=Be]": 222,
+      "[C@H1]": 223,
+      "[\\C]": 224,
+      "[Ca]": 225,
+      "[Si+4]": 226,
+      "[PH1]": 227,
+      "[Nb]": 228,
+      "[Cr+3]": 229,
+      "[Branch1]": 230,
+      "[Hf+4]": 231,
+      "[=Nb]": 232,
+      "[P]": 233,
+      "[CH1]": 234,
+      "[Ag]": 235,
+      "[=C-1]": 236,
+      "[\\-Ring1]": 237,
+      "[C-1]": 238,
+      "[C@@]": 239,
+      "[/Cl]": 240,
+      "[Cu+2]": 241,
+      "[Cd]": 242,
+      "[CaH2]": 243,
+      "[=Co]": 244,
+      "[/C@@]": 245,
+      "[Li]": 246,
+      "[Ti]": 247,
+      "[=C]": 248,
+      "[P+1]": 249,
+      "[=CH0]": 250,
+      "[=Y]": 251,
+      "[V+2]": 252,
+      "[SiH1]": 253,
+      "[V]": 254,
+      "[La+3]": 255,
+      "[Ru+3]": 256,
+      "[=Cu]": 257,
+      "[C@]": 258,
+      "[Fe+3]": 259,
+      "[Te]": 260,
+      "[=Zn]": 261,
+      "[Bi]": 262,
+      "[Br]": 263,
+      "[/C@H1]": 264,
+      "[Mn]": 265,
+      "[SH0]": 266,
+      "[Pt+4]": 267,
+      "[As]": 268,
+      "[/NH1]": 269,
+      "[Nop]": 270,
+      "[Ring3]": 271,
+      "[#Branch3]": 272,
+      "[=Branch3]": 273,
+      "[\\C@]": 274,
+      "[#N+1]": 275,
+      "[=Ring3]": 276
     }
   }
 }

tokenizer_config.json CHANGED Viewed

@@ -1,12 +1,10 @@
 {
   "cls_token": "[bos]",
   "mask_token": "[mask]",
-  "model_max_length": 427,
-  "name_or_path": "tokenizer",
   "pad_token": "[nop]",
   "padding_side": "right",
   "sep_token": "[eos]",
-  "special_tokens_map_file": "tokenizer/special_tokens_map.json",
   "tokenizer_class": "PreTrainedTokenizerFast",
   "truncation_side": "right",
   "unk_token": "[unk]"

 {
   "cls_token": "[bos]",
   "mask_token": "[mask]",
+  "model_max_length": 135,
   "pad_token": "[nop]",
   "padding_side": "right",
   "sep_token": "[eos]",
   "tokenizer_class": "PreTrainedTokenizerFast",
   "truncation_side": "right",
   "unk_token": "[unk]"