Upload folder using huggingface_hub

README.md

@@ -47,4 +47,8 @@ tokens = tokenizer.encode("Hello, world!")
 ## Sample Encoding
 | Text | Tokens | Token IDs |
 |------|--------|-----------|
-| `yirmi iki+dokuz=otuz bir\ntwenty two+nine=thirty one` | `y, i, r, m, i, Ġ, i, k, i, +, d, o, k, u, z, =, o, t, u, z` | `91, 75, 84, 79, 75, 223, 75, 77, 75, 3, 70, 81, 77, 87, 92, 4, 81, 86, 87, 92` |
+| `yirmi iki+dokuz=otuz bir\ntwenty two+nine=thirty one` | `y, i, r, m, i, Ġ, i, k, i, +, d, o, k, u, z, =, o, t, u, z` | `91, 75, 84, 79, 75, 223, 75, 77, 75, 13, 70, 81, 77, 87, 92, 31, 81, 86, 87, 92` |
+
+Command used to create this tokenizer:
+```bash
+['/home/gsa/tokenizers2/flexitok/tokenizer_training/train_tokenizers.py', 'algorithm=bpe', 'vocab_size=2000', 'langs=[spa_Latn]', 'data_dir=/scratch/gsa/data/multilingual-addition/', 'output_dir=/scratch/gsa/trained_tokenizers/multilingual_addition', 'pretokenizer=custom:addition', 'number_handling=ltr_3digit', 'add_numbers=false', 'handle_contractions=false', 'unicode_normalization=nfc', 'use_byte_level_regex=false', 'byte_fallback=false', 'strip_zero_width=false', 'cjk_char_split=false', 'add_cjk_chars=false', 'max_lines=-1', 'test_string=yirmi iki+dokuz=otuz bir\\ntwenty two+nine=thirty one', 'hf.publish_to_hf=true', 'hf_repo_prefix=flexitok/', 'hf.hf_repo_id=flexitok/maddition_spa_Latn_2000', 'hf.collections=[flexitok/multilingual_addition_tokenizers]']

special_tokens_map.json

@@ -1,8 +1,4 @@
 {
-  "additional_special_tokens": [
-    "+",
-    "="
-  ],
   "bos_token": "<s>",
   "eos_token": "</s>",
   "pad_token": "<pad>"

tokenizer.json

@@ -29,24 +29,6 @@
       "rstrip": false,
       "normalized": false,
       "special": true
-    },
-    {
-      "id": 3,
-      "content": "+",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 4,
-      "content": "=",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
     }
   ],
   "normalizer": {
@@ -99,35 +81,35 @@
       "<s>": 0,
       "</s>": 1,
       "<pad>": 2,
-      "+": 3,
-      "=": 4,
-      "!": 5,
-      "\"": 6,
-      "#": 7,
-      "$": 8,
-      "%": 9,
-      "&": 10,
-      "'": 11,
-      "(": 12,
-      ")": 13,
-      "*": 14,
-      ",": 15,
-      "-": 16,
-      ".": 17,
-      "/": 18,
-      "0": 19,
-      "1": 20,
-      "2": 21,
-      "3": 22,
-      "4": 23,
-      "5": 24,
-      "6": 25,
-      "7": 26,
-      "8": 27,
-      "9": 28,
-      ":": 29,
-      ";": 30,
-      "<": 31,
+      "!": 3,
+      "\"": 4,
+      "#": 5,
+      "$": 6,
+      "%": 7,
+      "&": 8,
+      "'": 9,
+      "(": 10,
+      ")": 11,
+      "*": 12,
+      "+": 13,
+      ",": 14,
+      "-": 15,
+      ".": 16,
+      "/": 17,
+      "0": 18,
+      "1": 19,
+      "2": 20,
+      "3": 21,
+      "4": 22,
+      "5": 23,
+      "6": 24,
+      "7": 25,
+      "8": 26,
+      "9": 27,
+      ":": 28,
+      ";": 29,
+      "<": 30,
+      "=": 31,
       ">": 32,
       "?": 33,
       "@": 34,

tokenizer_config.json

@@ -23,28 +23,8 @@
       "rstrip": false,
       "single_word": false,
       "special": true
-    },
-    "3": {
-      "content": "+",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "4": {
-      "content": "=",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
     }
   },
-  "additional_special_tokens": [
-    "+",
-    "="
-  ],
   "bos_token": "<s>",
   "clean_up_tokenization_spaces": false,
   "eos_token": "</s>",

vocab.json

@@ -1,374 +1,374 @@
 {
-  "os": 262,
-  "veinticinco": 352,
-  "ß": 158,
-  "O": 49,
-  "tenta": 294,
-  "och": 271,
-  "ć": 198,
-  "đ": 208,
+  "ô": 179,
+  "F": 40,
+  "cien": 369,
+  "ã": 162,
+  "cat": 335,
+  "é": 168,
+  "entos": 263,
+  "ê": 169,
+  "Ì": 139,
+  "Ü": 155,
+  "diez": 356,
+  "x": 90,
+  "ç": 166,
+  "Ł": 256,
+  "Ń": 258,
+  "í": 172,
+  "X": 58,
+  "³": 114,
+  ":": 28,
+  "ĕ": 212,
+  "ł": 257,
+  "é": 330,
+  "\\": 62,
+  "dos": 281,
+  "y": 91,
+  "ð": 175,
+  "W": 57,
+  "n": 80,
+  "9": 27,
+  "ï": 174,
+  "î": 173,
+  "ij": 242,
+  "Ħ": 229,
+  "/": 17,
+  "~": 96,
+  "Ą": 195,
+  "¶": 117,
+  "3": 21,
+  "cinco": 324,
+  "veintitré": 355,
+  "Ļ": 250,
+  "4": 22,
+  "Č": 203,
+  "Ý": 156,
+  "ė": 214,
   "D": 38,
-  "¸": 119,
-  "Â": 129,
-  "b": 68,
-  "Ģ": 225,
-  "co": 321,
-  "mil": 285,
-  "m": 79,
+  "at": 334,
+  "¥": 101,
   "?": 33,
-  "ciento": 320,
-  "a": 67,
-  "nce": 338,
-  "t": 86,
-  "µ": 116,
-  "Ğ": 221,
+  "Ĵ": 243,
+  "tr": 266,
+  "]": 63,
+  ">": 32,
   "catorce": 367,
+  "©": 105,
+  "seis": 282,
+  "c": 69,
+  "À": 127,
+  "ę": 216,
+  "à": 159,
+  "o": 81,
+  "ķ": 246,
   "Ð": 143,
-  "cat": 335,
-  "į": 238,
-  "Ĕ": 211,
-  "veintiuno": 351,
-  "M": 47,
-  "veinti": 327,
-  "¥": 101,
-  "ą": 196,
+  "cuarenta": 309,
+  "m": 79,
+  "č": 204,
+  "İ": 239,
+  "I": 43,
+  "§": 103,
+  "vecientos": 301,
+  "[": 61,
+  "Ù": 152,
+  "i": 75,
+  "w": 89,
+  "enta": 265,
+  "û": 186,
+  "ocientos": 277,
+  "´": 115,
+  "sesenta": 298,
+  "³s": 342,
+  "ď": 206,
+  "nce": 338,
+  "N": 48,
   "ä": 163,
-  "cuatrocientos": 311,
-  "Ì": 139,
-  "^": 64,
-  "Č": 203,
-  "Ĭ": 235,
-  "Ļ": 250,
-  "h": 74,
-  "¿": 126,
-  "½": 124,
-  "ù": 184,
-  "doscientos": 312,
+  "æ": 165,
+  "Ī": 233,
+  "+": 13,
+  "siete": 322,
+  "die": 328,
+  "è": 167,
+  "a": 67,
+  "r": 84,
+  "Ĥ": 227,
+  "¯": 110,
+  "×": 150,
+  "ą": 196,
+  "ě": 218,
+  "Q": 51,
+  "P": 50,
+  "</s>": 1,
+  "J": 44,
+  "Ā": 191,
+  "ĭ": 236,
   "dieciséis": 360,
-  "´": 115,
-  "c": 69,
-  ",": 15,
-  "ento": 319,
-  "ñ": 176,
-  "ocientos": 277,
-  "i": 75,
-  "ci": 261,
-  "E": 39,
-  "ę": 216,
-  "s": 85,
-  "once": 340,
-  "F": 40,
-  "_": 65,
+  "¢": 98,
   "oc": 269,
-  "sesenta": 298,
-  "Ł": 256,
-  "9": 28,
-  "uno": 323,
-  "ã": 162,
+  "=": 31,
+  "qu": 288,
+  "L": 46,
+  "ì": 171,
+  "R": 52,
+  "Ĩ": 231,
+  "veinticuatro": 353,
+  "ú": 185,
+  "nueve": 326,
+  "¿": 126,
+  "renta": 292,
+  "²": 113,
+  "2": 20,
+  "Ĺ": 248,
+  "H": 42,
+  "il": 284,
+  "B": 36,
   "setenta": 299,
-  "d": 70,
-  "cinueve": 345,
-  "é": 330,
-  "¶": 117,
-  "veintiocho": 350,
-  "te": 278,
+  "ÿ": 190,
+  "Ã": 130,
+  "Ñ": 144,
+  "s": 85,
+  "Ě": 217,
+  "once": 340,
+  "ā": 192,
+  "Ø": 151,
+  "·": 118,
+  "ie": 286,
   "Õ": 148,
-  "cero": 371,
-  "trescientos": 314,
-  "+": 3,
-  "K": 45,
-  "¦": 102,
-  "int": 276,
-  "nt": 259,
-  "tr": 266,
-  "doc": 337,
-  "¹": 120,
-  "en": 368,
-  "or": 339,
+  "treinta": 318,
+  "setecientos": 317,
+  "Ĭ": 235,
+  "veint": 302,
+  "uno": 323,
+  "k": 77,
+  "V": 56,
+  "ı": 240,
   "A": 35,
-  "veintisiete": 361,
+  "co": 321,
+  "Đ": 207,
+  "ĥ": 228,
+  "ľ": 253,
+  "trece": 344,
+  "se": 267,
+  "doce": 364,
+  "dÃ": 336,
+  "@": 34,
+  "Û": 154,
+  "U": 55,
   "f": 72,
+  "ī": 234,
+  "ochocientos": 305,
+  "ientos": 291,
+  ",": 14,
+  "t": 86,
+  "6": 24,
+  "K": 45,
+  "¤": 100,
+  "ª": 106,
+  "Ú": 153,
+  "»": 122,
+  "ĺ": 249,
+  "Ŀ": 254,
+  "Ę": 215,
+  "e": 71,
+  "±": 112,
   "ueve": 295,
-  "cinco": 324,
-  "â": 161,
-  "</s>": 1,
-  "`": 66,
-  "ô": 179,
-  "qu": 288,
-  "tre": 273,
-  "J": 44,
-  "4": 23,
-  "§": 103,
-  "iete": 315,
-  "Ġ": 223,
-  "Û": 154,
-  "R": 52,
-  "×": 150,
-  "at": 334,
-  "¼": 123,
-  "ó": 178,
-  "/": 18,
-  "ï": 174,
-  "treint": 308,
-  "ċ": 202,
-  "éis": 333,
-  "o": 81,
-  "veintiséis": 362,
-  "-": 16,
-  "Ą": 195,
-  "x": 90,
-  "1": 20,
-  "Í": 140,
-  "@": 34,
+  "*": 12,
+  "á": 160,
+  "E": 39,
+  "<pad>": 2,
+  "5": 23,
+  "mil": 285,
+  "ĵ": 244,
+  "Â": 129,
+  "1": 19,
+  "p": 82,
+  "¦": 102,
+  "venta": 296,
+  "int": 276,
+  "Ă": 193,
+  "Ć": 197,
+  "ē": 210,
+  "veintitrés": 366,
   "Ė": 213,
-  "Ó": 146,
-  "=": 4,
-  "S": 53,
-  "veintinueve": 349,
-  "Ĵ": 243,
-  "G": 41,
-  "is": 275,
-  "séis": 341,
-  "cu": 268,
-  "z": 92,
-  "veintis": 332,
-  "®": 109,
-  "Ø": 151,
-  "ĩ": 232,
-  "Z": 60,
-  "ľ": 253,
-  "Ç": 134,
-  "Ò": 145,
-  "ĺ": 249,
-  "ent": 260,
-  "č": 204,
-  "3": 22,
-  "noventa": 306,
+  "te": 278,
+  "¼": 123,
+  "£": 99,
+  "å": 164,
   "seiscientos": 313,
+  "cator": 363,
+  "ĉ": 200,
+  "veintinueve": 349,
+  "ý": 188,
+  "ħ": 230,
+  "ce": 329,
+  "dieci": 331,
+  "ocho": 303,
+  "`": 66,
+  "diecinueve": 357,
+  "ciento": 320,
   "diecisiete": 359,
-  "ļ": 251,
-  "Ĥ": 227,
-  "inueve": 346,
-  "Ê": 137,
-  "ÿ": 190,
-  "©": 105,
-  "p": 82,
-  "Ĝ": 219,
-  ")": 13,
-  "Ô": 147,
-  "trece": 344,
-  "Æ": 133,
-  "ė": 214,
-  "²": 113,
-  "diez": 356,
-  "~": 96,
-  "6": 25,
-  "l": 78,
-  "veinticuatro": 353,
-  "eve": 290,
-  "'": 11,
-  "º": 121,
-  "Ý": 156,
+  "h": 74,
+  "cero": 371,
   "Ķ": 245,
-  "<pad>": 2,
-  "quinientos": 316,
-  "ientos": 291,
-  "U": 55,
-  "senta": 293,
-  "ò": 177,
-  "Ã": 130,
-  "N": 48,
-  "ü": 187,
-  "V": 56,
-  "doce": 364,
-  "¨": 104,
-  "veintitrés": 366,
-  "Ù": 152,
-  "W": 57,
-  "Ī": 233,
-  "veint": 302,
-  "ç": 166,
-  "quince": 347,
-  "I": 43,
-  "w": 89,
-  "ë": 170,
-  "ö": 181,
-  "³": 114,
-  "ĵ": 244,
-  "k": 77,
-  "!": 5,
-  "Ď": 205,
-  "g": 73,
-  ".": 17,
-  "se": 267,
-  "siete": 322,
-  "y": 91,
+  "ochenta": 304,
+  "novecientos": 307,
+  "%": 7,
+  "is": 275,
+  "Ğ": 221,
+  "veintiocho": 350,
+  "µ": 116,
+  "¹": 120,
+  "¾": 125,
+  "trescientos": 314,
+  "â": 161,
+  "¡": 97,
+  "Ĕ": 211,
   "ve": 270,
-  "Þ": 157,
-  "C": 37,
-  "quin": 289,
-  "cua": 274,
-  "Ċ": 201,
-  "¯": 110,
-  "Ā": 191,
-  "cator": 363,
-  "P": 50,
-  "ĝ": 220,
-  "ie": 286,
-  ";": 30,
-  "ĕ": 212,
-  "è": 167,
-  "³s": 342,
-  "·": 118,
-  "cien": 369,
-  "e": 71,
-  "£": 99,
-  "seis": 282,
-  "Ö": 149,
-  "ĥ": 228,
-  ":": 29,
-  "L": 46,
-  "dos": 281,
-  "7": 26,
-  "nueve": 326,
-  "à": 159,
-  "*": 14,
-  "j": 76,
+  "noventa": 306,
   "veinte": 348,
-  "¡": 97,
-  "ocho": 303,
-  "õ": 180,
-  "ce": 329,
-  "é": 168,
-  "Ü": 155,
-  "Î": 141,
-  "È": 135,
-  "ď": 206,
-  "ā": 192,
-  "2": 21,
-  "ĭ": 236,
-  "¢": 98,
-  "cuatro": 325,
-  "setecientos": 317,
+  "Ď": 205,
+  "Á": 128,
+  "ĝ": 220,
+  "'": 9,
+  "veintiséis": 362,
   "T": 54,
-  "ı": 240,
-  "Ľ": 252,
-  "Ń": 258,
-  "ro": 370,
-  "dieciocho": 358,
-  "entos": 263,
-  "ú": 185,
+  "quinientos": 316,
+  "tre": 273,
+  "þ": 189,
+  "veinticinco": 352,
+  "!": 3,
+  "j": 76,
+  "veintis": 332,
+  "cincuenta": 310,
+  "Ģ": 225,
+  "C": 37,
+  "or": 339,
+  "Ê": 137,
+  "}": 95,
+  "tenta": 294,
+  "u": 87,
+  "ć": 198,
+  "quince": 347,
+  "(": 10,
+  "ġ": 224,
   "÷": 182,
-  "ð": 175,
-  "]": 63,
-  "%": 9,
-  "Ĩ": 231,
-  "{": 93,
-  "ħ": 230,
-  "#": 7,
-  "Ħ": 229,
-  "Ē": 209,
+  "7": 25,
+  "z": 92,
+  "&": 8,
+  "ë": 170,
+  "veintidÃ": 354,
   "¬": 108,
-  "å": 164,
-  "Ñ": 144,
-  "cincuenta": 310,
-  "ě": 218,
-  "treinta": 318,
-  "Ă": 193,
-  "$": 8,
+  "v": 88,
+  "Ç": 134,
   "cuatr": 280,
-  "8": 27,
-  "diecinueve": 357,
-  "¤": 100,
-  "Ŀ": 254,
-  "Ë": 138,
-  "ª": 106,
-  "«": 107,
-  "Y": 59,
-  "Đ": 207,
-  "tres": 283,
-  "Ĺ": 248,
   "°": 111,
-  "Ä": 131,
-  "enta": 265,
-  "|": 94,
-  "ij": 242,
-  "ī": 234,
-  "ŀ": 255,
-  "X": 58,
-  "Q": 51,
+  "nt": 259,
+  "cuenta": 300,
+  "^": 64,
+  ")": 11,
+  "S": 53,
+  "quin": 289,
+  "cuatrocientos": 311,
+  "cientos": 264,
   "Å": 132,
-  "ġ": 224,
-  "}": 95,
-  "(": 12,
-  "±": 112,
-  "0": 19,
-  "v": 88,
-  "Ę": 215,
-  "Á": 128,
-  "ì": 171,
-  "veintidÃ": 354,
-  "ĉ": 200,
-  "r": 84,
-  "Ć": 197,
-  "İ": 239,
-  "B": 36,
+  "tres": 283,
+  "Y": 59,
+  "ent": 260,
+  "treint": 308,
   "sete": 297,
-  "Į": 237,
-  "ochocientos": 305,
-  "æ": 165,
-  "þ": 189,
-  "cuenta": 300,
-  "É": 136,
-  "5": 24,
+  "Ē": 209,
+  "º": 121,
+  "cin": 279,
+  "õ": 180,
+  "G": 41,
+  "doc": 337,
+  "Ľ": 252,
+  "l": 78,
+  "cu": 268,
+  "-": 15,
+  "đ": 208,
+  "ß": 158,
+  "dieciocho": 358,
+  "en": 368,
+  "Z": 60,
+  "O": 49,
+  "Þ": 157,
+  "_": 65,
   "no": 272,
-  "ĸ": 247,
-  "ý": 188,
-  "Ě": 217,
-  "Ĉ": 199,
-  "&": 10,
-  "Ú": 153,
-  "î": 173,
-  "vecientos": 301,
-  "ă": 194,
-  "\\": 62,
-  "veintitré": 355,
-  "À": 127,
-  "[": 61,
-  "IJ": 241,
-  "<": 31,
-  "ķ": 246,
+  "Ó": 146,
+  "iete": 315,
+  "Ë": 138,
+  "Ġ": 223,
+  "veintisiete": 361,
+  "ļ": 251,
+  "eve": 290,
+  "doscientos": 312,
+  "b": 68,
+  ";": 29,
+  "Ò": 145,
+  "«": 107,
+  "ó": 178,
+  "ü": 187,
+  "cuatro": 325,
+  "ro": 370,
+  "8": 26,
+  "¸": 119,
+  "Ö": 149,
+  "in": 287,
+  "Í": 140,
+  "¨": 104,
+  "ù": 184,
+  "q": 83,
   "Ï": 142,
-  "n": 80,
-  "\"": 6,
-  "H": 42,
-  "cuarenta": 309,
-  "¾": 125,
   "ģ": 226,
-  "í": 172,
-  "in": 287,
-  "cin": 279,
-  "novecientos": 307,
+  "į": 238,
+  "ŀ": 255,
+  "d": 70,
+  "ö": 181,
+  ".": 16,
+  "os": 262,
   "veintidós": 365,
-  ">": 32,
-  "ē": 210,
-  "ł": 257,
-  "il": 284,
-  "cientos": 264,
-  "renta": 292,
-  "ø": 183,
-  "q": 83,
-  "ğ": 222,
-  "u": 87,
-  "tré": 343,
-  "»": 122,
-  "venta": 296,
+  "È": 135,
+  "ĩ": 232,
+  "ñ": 176,
+  "|": 94,
+  "Æ": 133,
+  "{": 93,
+  "Į": 237,
   "<s>": 0,
-  "á": 160,
-  "ê": 169,
-  "die": 328,
-  "û": 186,
-  "ochenta": 304,
-  "dÃ": 336,
-  "dieci": 331
+  "½": 124,
+  "séis": 341,
+  "<": 30,
+  "Î": 141,
+  "tré": 343,
+  "senta": 293,
+  "ğ": 222,
+  "ĸ": 247,
+  "É": 136,
+  "\"": 4,
+  "#": 5,
+  "inueve": 346,
+  "0": 18,
+  "Ċ": 201,
+  "cua": 274,
+  "M": 47,
+  "Ä": 131,
+  "och": 271,
+  "ø": 183,
+  "ci": 261,
+  "Ĝ": 219,
+  "g": 73,
+  "veinti": 327,
+  "éis": 333,
+  "IJ": 241,
+  "Ĉ": 199,
+  "ò": 177,
+  "ento": 319,
+  "Ô": 147,
+  "ċ": 202,
+  "ă": 194,
+  "$": 6,
+  "veintiuno": 351,
+  "®": 109,
+  "cinueve": 345
 }