Upload folder using huggingface_hub

README.md

@@ -47,4 +47,8 @@ tokens = tokenizer.encode("Hello, world!")
 ## Sample Encoding
 | Text | Tokens | Token IDs |
 |------|--------|-----------|
-| `yirmi iki+dokuz=otuz bir\ntwenty two+nine=thirty one` | `y, i, r, m, i, Ġ, i, k, i, +, d, o, k, u, z, =, o, t, u, z` | `91, 75, 84, 79, 75, 223, 75, 77, 75, 3, 70, 81, 77, 87, 92, 4, 81, 86, 87, 92` |
+| `yirmi iki+dokuz=otuz bir\ntwenty two+nine=thirty one` | `y, i, r, m, i, Ġ, i, k, i, +, d, o, k, u, z, =, o, t, u, z` | `91, 75, 84, 79, 75, 223, 75, 77, 75, 13, 70, 81, 77, 87, 92, 31, 81, 86, 87, 92` |
+
+Command used to create this tokenizer:
+```bash
+['/home/gsa/tokenizers2/flexitok/tokenizer_training/train_tokenizers.py', 'algorithm=bpe', 'vocab_size=2000', 'langs=[pol_Latn]', 'data_dir=/scratch/gsa/data/multilingual-addition/', 'output_dir=/scratch/gsa/trained_tokenizers/multilingual_addition', 'pretokenizer=custom:addition', 'number_handling=ltr_3digit', 'add_numbers=false', 'handle_contractions=false', 'unicode_normalization=nfc', 'use_byte_level_regex=false', 'byte_fallback=false', 'strip_zero_width=false', 'cjk_char_split=false', 'add_cjk_chars=false', 'max_lines=-1', 'test_string=yirmi iki+dokuz=otuz bir\\ntwenty two+nine=thirty one', 'hf.publish_to_hf=true', 'hf_repo_prefix=flexitok/', 'hf.hf_repo_id=flexitok/maddition_pol_Latn_2000', 'hf.collections=[flexitok/multilingual_addition_tokenizers]']

special_tokens_map.json

@@ -1,8 +1,4 @@
 {
-  "additional_special_tokens": [
-    "+",
-    "="
-  ],
   "bos_token": "<s>",
   "eos_token": "</s>",
   "pad_token": "<pad>"

tokenizer.json

@@ -29,24 +29,6 @@
       "rstrip": false,
       "normalized": false,
       "special": true
-    },
-    {
-      "id": 3,
-      "content": "+",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
-    },
-    {
-      "id": 4,
-      "content": "=",
-      "single_word": false,
-      "lstrip": false,
-      "rstrip": false,
-      "normalized": false,
-      "special": true
     }
   ],
   "normalizer": {
@@ -99,35 +81,35 @@
       "<s>": 0,
       "</s>": 1,
       "<pad>": 2,
-      "+": 3,
-      "=": 4,
-      "!": 5,
-      "\"": 6,
-      "#": 7,
-      "$": 8,
-      "%": 9,
-      "&": 10,
-      "'": 11,
-      "(": 12,
-      ")": 13,
-      "*": 14,
-      ",": 15,
-      "-": 16,
-      ".": 17,
-      "/": 18,
-      "0": 19,
-      "1": 20,
-      "2": 21,
-      "3": 22,
-      "4": 23,
-      "5": 24,
-      "6": 25,
-      "7": 26,
-      "8": 27,
-      "9": 28,
-      ":": 29,
-      ";": 30,
-      "<": 31,
+      "!": 3,
+      "\"": 4,
+      "#": 5,
+      "$": 6,
+      "%": 7,
+      "&": 8,
+      "'": 9,
+      "(": 10,
+      ")": 11,
+      "*": 12,
+      "+": 13,
+      ",": 14,
+      "-": 15,
+      ".": 16,
+      "/": 17,
+      "0": 18,
+      "1": 19,
+      "2": 20,
+      "3": 21,
+      "4": 22,
+      "5": 23,
+      "6": 24,
+      "7": 25,
+      "8": 26,
+      "9": 27,
+      ":": 28,
+      ";": 29,
+      "<": 30,
+      "=": 31,
       ">": 32,
       "?": 33,
       "@": 34,

tokenizer_config.json

@@ -23,28 +23,8 @@
       "rstrip": false,
       "single_word": false,
       "special": true
-    },
-    "3": {
-      "content": "+",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "4": {
-      "content": "=",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
     }
   },
-  "additional_special_tokens": [
-    "+",
-    "="
-  ],
   "bos_token": "<s>",
   "clean_up_tokenization_spaces": false,
   "eos_token": "</s>",

vocab.json

@@ -1,348 +1,348 @@
 {
-  "Ĩ": 231,
-  "ç": 166,
-  "Į": 237,
-  "5": 24,
-  "Z": 60,
-  "trzysta": 314,
-  "\"": 6,
-  "Ł": 256,
-  "q": 83,
-  "Ñ": 144,
-  "ieÅĽcie": 307,
-  "dziewiÄĻtnaÅĽcie": 335,
-  "Ħ": 229,
-  ",": 15,
-  "Ö": 149,
-  "e": 71,
-  "ÿ": 190,
-  "ÅĽc": 274,
-  "Ð": 143,
-  "1": 20,
-  "</s>": 1,
-  "¨": 104,
-  "cz": 277,
-  "Ď": 205,
-  "Ķ": 245,
-  "0": 19,
-  "P": 50,
+  "Ļ": 250,
+  "siedemdziesiÄħt": 317,
+  "set": 271,
   "jede": 306,
-  "å": 164,
+  "ĉ": 200,
+  "Ć": 197,
+  "Äĩ": 263,
+  "è": 167,
+  "5": 23,
+  "Þ": 157,
+  "Ï": 142,
+  "piÄĻtnaÅĽcie": 331,
+  "siÄħc": 303,
+  "cztery": 301,
+  "¸": 119,
+  "ċ": 202,
+  "czterysta": 325,
+  "Ú": 153,
+  "´": 115,
+  "İ": 239,
+  "ie": 259,
+  "ķ": 246,
+  "ġ": 224,
+  "jedenaÅĽcie": 342,
   "szeÅĽÄĩdziesiÄħt": 323,
-  "jeden": 326,
-  "ă": 194,
-  "czterdzieÅĽci": 310,
-  "ter": 283,
-  "dzieÅĽc": 286,
-  "G": 41,
-  "r": 84,
-  "dziesiÄ": 334,
-  "þ": 189,
-  "siedem": 293,
-  "'": 11,
+  "·": 118,
+  "@": 34,
+  "ù": 184,
+  "M": 47,
+  "í": 172,
+  "ë": 170,
+  "0": 18,
+  "Ŀ": 254,
+  "]": 63,
   "ô": 179,
-  "Ç": 134,
-  "Ã": 130,
-  "sze": 290,
-  "dziewiÄĻÄĩset": 320,
-  "×": 150,
+  "dwieÅĽcie": 308,
+  "=": 31,
+  "¬": 108,
+  "osiemnaÅĽcie": 337,
+  "dziewiÄĻÄĩ": 296,
+  "ĺ": 249,
+  "Ô": 147,
+  "piÄĻÄĩset": 318,
+  "A": 35,
+  "Ľ": 252,
+  "ÅĽc": 274,
+  "ø": 183,
   "L": 46,
-  "osiem": 289,
-  "Ä": 131,
-  "¹": 120,
+  "osie": 279,
+  "siÄħ": 266,
   "Ê": 137,
-  "¦": 102,
-  "T": 54,
-  "czter": 288,
-  "©": 105,
-  "siÄħc": 303,
-  "j": 76,
-  "piÄĻÄĩ": 294,
-  "²": 113,
-  "\\": 62,
-  "zero": 345,
-  "ty": 302,
-  "Ô": 147,
+  "I": 43,
+  "ĻÄĩ": 333,
+  "Ĕ": 211,
+  "*": 12,
+  "r": 84,
+  "Ó": 146,
+  "iÄĻ": 268,
+  "N": 48,
+  "siÄ": 265,
+  "ß": 158,
+  "ĭ": 236,
+  "8": 26,
+  "&": 8,
+  "Ç": 134,
+  "n": 80,
+  "iÄ": 260,
+  "6": 24,
+  "dziesiÄĻÄĩ": 343,
+  "tnaÅĽcie": 329,
+  "e": 71,
+  "û": 186,
+  "Ķ": 245,
+  "«": 107,
   "Õ": 148,
-  "de": 275,
+  "dziewiÄĻÄĩset": 320,
+  "Z": 60,
+  "<": 30,
+  "trz": 284,
+  "G": 41,
+  "s": 85,
+  "B": 36,
   "osiemset": 311,
-  "ĩ": 232,
-  "Þ": 157,
-  "Ċ": 201,
-  "»": 122,
-  "piÄĻÄĩset": 318,
-  "î": 173,
   "Ì": 139,
-  "sie": 267,
-  "Ú": 153,
-  "_": 65,
-  "Ī": 233,
-  "dwieÅĽcie": 308,
-  "È": 135,
-  "k": 77,
-  ".": 17,
-  "«": 107,
-  "Ĥ": 227,
-  "+": 3,
-  "&": 10,
+  "P": 50,
+  "î": 173,
+  "°": 111,
+  "ê": 169,
+  "½": 124,
+  "+": 13,
+  "ede": 305,
+  "IJ": 241,
   "F": 40,
-  "ü": 187,
-  "set": 271,
-  "b": 68,
-  "siedemdziesiÄħt": 317,
-  "trzy": 291,
-  "czterysta": 325,
-  "szesnaÅĽcie": 338,
-  "Ć": 197,
-  "K": 45,
-  "x": 90,
-  "¥": 101,
-  "dwadzieÅĽcia": 324,
+  "1": 19,
+  "ă": 194,
+  "siedemnaÅĽcie": 340,
+  "dziesiÄħt": 273,
+  "trzysta": 314,
+  "Ę": 215,
   "ĸ": 247,
-  "Đ": 207,
-  "Č": 203,
-  ">": 32,
-  ")": 13,
-  "ē": 210,
-  "h": 74,
-  "¢": 98,
-  "đ": 208,
-  "Ó": 146,
-  "Ë": 138,
-  "À": 127,
-  "v": 88,
-  "·": 118,
-  "d": 70,
-  "ï": 174,
-  "N": 48,
-  "f": 72,
-  "ĝ": 220,
-  "Ĭ": 235,
-  "$": 8,
-  "o": 81,
-  "¸": 119,
-  "¡": 97,
-  "ÅĽ": 264,
-  "é": 168,
-  "~": 96,
-  "I": 43,
-  "n": 80,
-  "er": 276,
-  "a": 67,
-  "ã": 162,
-  "<": 31,
-  "l": 78,
-  "osiemdziesiÄħt": 312,
-  "â": 161,
-  "ÅĽÄĩ": 295,
-  "dwanaÅĽcie": 341,
-  "R": 52,
+  "Ĵ": 243,
+  "¨": 104,
+  "}": 95,
+  "ı": 240,
+  "Ħ": 229,
   "á": 160,
-  "S": 53,
-  "siÄ": 265,
-  "ĭ": 236,
-  "H": 42,
-  "V": 56,
-  "E": 39,
-  "¼": 123,
-  "<s>": 0,
-  "ê": 169,
-  "8": 27,
-  "J": 44,
-  "dzie": 262,
-  "U": 55,
-  "Ń": 258,
-  "jedenaÅĽcie": 342,
   "Ġ": 223,
-  "ą": 196,
-  "ä": 163,
-  "c": 69,
-  "ď": 206,
-  "cztery": 301,
-  "Y": 59,
-  "¶": 117,
-  "X": 58,
-  "Ă": 193,
-  "7": 26,
-  "¯": 110,
-  "iÄĻ": 268,
-  "±": 112,
+  "³": 114,
+  "dwanaÅĽcie": 341,
+  "ÅĽcie": 300,
+  "à": 159,
+  "o": 81,
+  "À": 127,
+  "ý": 188,
+  "ī": 234,
+  "szesnaÅĽcie": 338,
+  "j": 76,
+  "É": 136,
+  "ü": 187,
+  "trzydzieÅĽci": 313,
+  "ö": 181,
+  "Ł": 256,
+  "Ĺ": 248,
+  "w": 89,
+  "cz": 277,
+  "ter": 283,
+  "R": 52,
+  "piÄĻÄĩdziesiÄħt": 319,
+  "õ": 180,
+  "ÿ": 190,
+  ",": 14,
+  "W": 57,
   "?": 33,
-  "C": 37,
-  "IJ": 241,
-  "osie": 279,
-  "°": 111,
-  "snaÅĽcie": 332,
-  "¾": 125,
+  "ļ": 251,
+  "<pad>": 2,
+  "siedemset": 316,
+  "Ģ": 225,
+  "tysiÄħc": 304,
+  "[": 61,
+  "â": 161,
   "ð": 175,
-  "ie": 259,
-  "Ŀ": 254,
-  "ě": 218,
-  "ù": 184,
-  "`": 66,
-  "ĻÄĩ": 333,
-  "ø": 183,
-  "}": 95,
-  "Ï": 142,
-  "p": 82,
-  "ú": 185,
+  "sie": 267,
+  "czternaÅĽcie": 336,
+  "X": 58,
+  "÷": 182,
+  "D": 38,
+  "~": 96,
+  "Ĥ": 227,
+  "Ĭ": 235,
+  "3": 21,
+  "piÄĻÄĩ": 294,
+  "Y": 59,
+  "ì": 171,
+  "dziewiÄĻtnaÅĽcie": 335,
+  "st": 281,
+  "į": 238,
+  "dziesiÄħ": 272,
+  "å": 164,
+  "er": 276,
   "sz": 282,
-  "Å": 132,
-  "Ē": 209,
-  "û": 186,
-  "dwa": 298,
-  "£": 99,
-  "º": 121,
-  "Ý": 156,
-  "dziewiÄĻÄĩ": 296,
-  "ĕ": 212,
-  "[": 61,
-  "ĺ": 249,
-  "4": 23,
-  "Î": 141,
-  "ġ": 224,
-  "İ": 239,
-  "(": 12,
-  "@": 34,
-  "trz": 284,
-  "ª": 106,
-  "Ĺ": 248,
-  "z": 92,
+  "dziew": 285,
+  "¡": 97,
+  "Í": 140,
+  ";": 29,
+  "º": 121,
+  "sta": 299,
+  "'": 9,
+  "Ā": 191,
+  ">": 32,
+  "Ý": 156,
+  "Đ": 207,
+  "sze": 290,
+  "ÅĽ": 264,
+  "é": 168,
+  "ą": 196,
   "¿": 126,
-  "ĵ": 244,
-  "ę": 216,
-  "*": 14,
-  "ĥ": 228,
-  "ļ": 251,
-  "szeÅĽÄĩset": 322,
-  "et": 270,
-  "ł": 257,
-  "ā": 192,
-  "9": 28,
-  "ħ": 230,
-  "tysiÄħc": 304,
+  "czterdzieÅĽci": 310,
+  "Ñ": 144,
   "sto": 309,
-  "č": 204,
-  "Ĕ": 211,
-  "rz": 280,
-  "Ę": 215,
-  "dziesiÄħ": 272,
-  "dziesiÄħt": 273,
-  "dzieÅĽci": 292,
-  "ċ": 202,
-  "ó": 178,
-  "siedemnaÅĽcie": 340,
-  "ı": 240,
-  "ede": 305,
-  "/": 18,
-  "osiemnaÅĽcie": 337,
-  "µ": 116,
-  "¤": 100,
-  "ė": 214,
-  "=": 4,
-  "st": 281,
-  "u": 87,
+  "Ī": 233,
+  "Ċ": 201,
+  ")": 11,
+  "ÅĽÄĩ": 295,
+  "Ĝ": 219,
+  "zero": 345,
+  "4": 22,
+  "U": 55,
+  "×": 150,
+  "Ă": 193,
+  "J": 44,
   "Ù": 152,
-  "6": 25,
-  "÷": 182,
-  "ŀ": 255,
-  "w": 89,
+  "È": 135,
   "ò": 177,
+  "%": 7,
+  "/": 17,
+  "ty": 302,
+  "Á": 128,
+  "µ": 116,
+  "¦": 102,
+  "rz": 280,
+  "Ë": 138,
+  "p": 82,
+  "ĩ": 232,
+  "¯": 110,
+  "¤": 100,
+  "dzieÅĽcia": 315,
+  "Ä": 131,
+  "snaÅĽcie": 332,
+  "Ö": 149,
+  "et": 270,
+  "ě": 218,
+  "O": 49,
+  "C": 37,
+  "Ą": 195,
+  "trzy": 291,
+  "9": 27,
   "trzynaÅĽcie": 339,
-  "É": 136,
-  "Ĉ": 199,
-  "3": 22,
   "ć": 198,
+  "czter": 288,
+  "osiem": 289,
+  "dzieÅĽc": 286,
+  "Ě": 217,
+  "g": 73,
+  "K": 45,
+  "dwa": 298,
+  "b": 68,
+  "ieÅĽcie": 307,
+  "dzie": 262,
+  "h": 74,
+  "ā": 192,
+  "£": 99,
+  "Į": 237,
+  "<s>": 0,
+  "Ė": 213,
+  "Ń": 258,
+  "a": 67,
+  "-": 15,
+  "l": 78,
+  "ñ": 176,
+  "Č": 203,
+  "ę": 216,
+  "Ď": 205,
   "iÄĻÄĩ": 269,
-  "Ā": 191,
-  "Æ": 133,
-  "į": 238,
-  "siÄħ": 266,
-  "-": 16,
-  "dziew": 285,
-  "è": 167,
-  "]": 63,
-  "ģ": 226,
+  "Ē": 209,
+  "ij": 242,
+  "v": 88,
+  "ħ": 230,
+  "Ã": 130,
+  "ç": 166,
+  "ŀ": 255,
+  "2": 20,
+  "{": 93,
+  "$": 6,
   "ľ": 253,
-  "õ": 180,
-  "æ": 165,
-  "ĉ": 200,
-  "í": 172,
+  "jeden": 326,
+  "ï": 174,
+  "ĵ": 244,
+  "Â": 129,
+  "z": 92,
+  "®": 109,
+  "Ð": 143,
+  "u": 87,
+  "Å": 132,
+  "f": 72,
+  "k": 77,
+  "ã": 162,
+  "osiemdziesiÄħt": 312,
+  "!": 3,
+  "ĕ": 212,
+  "y": 91,
+  "ğ": 222,
+  "¹": 120,
+  "siedem": 293,
   "Û": 154,
-  "à": 159,
+  "dziesiÄ": 334,
+  "dz": 261,
+  "ł": 257,
+  "¾": 125,
+  "©": 105,
+  "Ğ": 221,
+  "T": 54,
+  "dw": 278,
+  "`": 66,
+  "đ": 208,
+  "Î": 141,
+  "ª": 106,
+  "ĝ": 220,
+  "č": 204,
+  "ú": 185,
+  "^": 64,
+  "i": 75,
+  "7": 25,
+  "¼": 123,
+  "S": 53,
   "zer": 344,
-  "ö": 181,
-  "¬": 108,
-  "W": 57,
-  "Ą": 195,
-  "naÅĽcie": 328,
-  "dzieÅĽcia": 315,
-  "{": 93,
-  "dziesiÄĻÄĩ": 343,
-  "piÄĻtnaÅĽcie": 331,
-  "Ĝ": 219,
-  "½": 124,
-  "Ģ": 225,
-  "Ø": 151,
-  "Ĵ": 243,
-  "ë": 170,
-  "!": 5,
-  "A": 35,
   "dziewiÄĻÄĩdziesiÄħt": 321,
-  "ñ": 176,
-  "y": 91,
-  "Äĩ": 263,
+  "Æ": 133,
+  "²": 113,
+  "¥": 101,
+  "ä": 163,
+  ":": 28,
+  "</s>": 1,
+  "æ": 165,
   "Ò": 145,
+  "naÅĽcie": 328,
+  "aÅĽcie": 327,
+  "m": 79,
+  "E": 39,
+  "Ĉ": 199,
+  "¢": 98,
+  "±": 112,
+  "ď": 206,
+  "c": 69,
+  "H": 42,
+  "ē": 210,
+  "»": 122,
   "Ü": 155,
-  "M": 47,
-  "siedemset": 316,
-  "ğ": 222,
-  "trzydzieÅĽci": 313,
-  "i": 75,
-  "§": 103,
-  "®": 109,
+  "þ": 189,
+  "x": 90,
+  ".": 16,
+  "¶": 117,
+  "dwadzieÅĽcia": 324,
+  "_": 65,
+  "Ĩ": 231,
+  "ė": 214,
+  "ĥ": 228,
+  "de": 275,
+  "(": 10,
+  "dzieÅĽci": 292,
   "Q": 51,
-  "<pad>": 2,
-  "|": 94,
-  "g": 73,
-  ":": 29,
-  "%": 9,
-  "Í": 140,
-  "sta": 299,
-  "D": 38,
-  "iÄĻtnaÅĽcie": 330,
-  "dz": 261,
-  "´": 115,
-  "B": 36,
-  "ý": 188,
-  "Á": 128,
-  "dw": 278,
-  "iÄ": 260,
-  "Ļ": 250,
-  "czternaÅĽcie": 336,
-  ";": 30,
-  "ß": 158,
-  "Ľ": 252,
-  "ī": 234,
-  "^": 64,
-  "#": 7,
-  "Ğ": 221,
+  "d": 70,
+  "szeÅĽÄĩset": 322,
   "t": 86,
-  "siede": 287,
-  "s": 85,
-  "O": 49,
-  "tnaÅĽcie": 329,
-  "³": 114,
-  "Ė": 213,
-  "Â": 129,
-  "ķ": 246,
-  "ij": 242,
+  "ó": 178,
+  "|": 94,
+  "ģ": 226,
   "szeÅĽÄĩ": 297,
-  "piÄĻÄĩdziesiÄħt": 319,
-  "Ě": 217,
-  "ì": 171,
-  "m": 79,
-  "ÅĽcie": 300,
-  "2": 21,
-  "aÅĽcie": 327
+  "siede": 287,
+  "\"": 4,
+  "§": 103,
+  "\\": 62,
+  "q": 83,
+  "Ø": 151,
+  "#": 5,
+  "iÄĻtnaÅĽcie": 330,
+  "V": 56
 }