Upload folder using huggingface_hub

README.md

@@ -1,14 +1,12 @@
 ---
 license: mit
 language:
-- und
+- und  # ISO 639-3 code or "und" if not identifiable
 tags:
 - tokenizer
 - bpe
 - flexitok
 - fineweb2
-datasets:
-- flexitok/mod-arithmetic
 ---
 
 # Byte-Level BPE Tokenizer: numeric (0K)

tokenizer.json

@@ -1 +1,578 @@
-{"<unk>": [0], "~SPECIAL~ALIGNED~BOS~SYMBOL~": [1], "</s>": [2], "<pad>": [3], "mod": [4], "=": [5], " ": [6], "0": [7], "1": [8], "2": [9], "3": [10], "4": [11], "5": [12], "6": [13], "7": [14], "8": [15], "9": [16], "10": [17], "11": [18], "12": [19], "13": [20], "14": [21], "15": [22], "16": [23], "17": [24], "18": [25], "19": [26], "20": [27], "21": [28], "22": [29], "23": [30], "24": [31], "25": [32], "26": [33], "27": [34], "28": [35], "29": [36], "30": [37], "31": [38], "32": [39], "33": [40], "34": [41], "35": [42], "36": [43], "37": [44], "38": [45], "39": [46], "40": [47], "41": [48], "42": [49], "43": [50], "44": [51], "45": [52], "46": [53], "47": [54], "48": [55], "49": [56], "50": [57], "51": [58], "52": [59], "53": [60], "54": [61], "55": [62], "56": [63], "57": [64], "58": [65], "59": [66], "60": [67], "61": [68], "62": [69], "63": [70], "64": [71], "65": [72], "66": [73], "67": [74], "68": [75], "69": [76], "70": [77], "71": [78], "72": [79], "73": [80], "74": [81], "75": [82], "76": [83], "77": [84], "78": [85], "79": [86], "80": [87], "81": [88], "82": [89], "83": [90], "84": [91], "85": [92], "86": [93], "87": [94], "88": [95], "89": [96], "90": [97], "91": [98], "92": [99], "93": [100], "94": [101], "95": [102], "96": [103], "97": [104], "98": [105], "99": [106]}
+{
+  "version": "1.0",
+  "truncation": null,
+  "padding": null,
+  "added_tokens": [
+    {
+      "id": 0,
+      "content": "<unk>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 1,
+      "content": "<s>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 2,
+      "content": "</s>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 3,
+      "content": "<pad>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 4,
+      "content": "mod",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 5,
+      "content": "=",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 6,
+      "content": " ",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    }
+  ],
+  "normalizer": null,
+  "pre_tokenizer": {
+    "type": "Sequence",
+    "pretokenizers": [
+      {
+        "type": "Split",
+        "pattern": {
+          "Regex": "\\p{N}{1,2}"
+        },
+        "behavior": "Isolated",
+        "invert": false
+      },
+      {
+        "type": "ByteLevel",
+        "add_prefix_space": false,
+        "trim_offsets": true,
+        "use_regex": false
+      }
+    ]
+  },
+  "post_processor": null,
+  "decoder": {
+    "type": "ByteLevel",
+    "add_prefix_space": true,
+    "trim_offsets": true,
+    "use_regex": true
+  },
+  "model": {
+    "type": "BPE",
+    "dropout": null,
+    "unk_token": "<unk>",
+    "continuing_subword_prefix": null,
+    "end_of_word_suffix": null,
+    "fuse_unk": false,
+    "byte_fallback": false,
+    "ignore_merges": false,
+    "vocab": {
+      "<unk>": 0,
+      "<s>": 1,
+      "</s>": 2,
+      "<pad>": 3,
+      "mod": 4,
+      "=": 5,
+      " ": 6,
+      "0": 7,
+      "1": 8,
+      "2": 9,
+      "3": 10,
+      "4": 11,
+      "5": 12,
+      "6": 13,
+      "7": 14,
+      "8": 15,
+      "9": 16,
+      "10": 17,
+      "11": 18,
+      "12": 19,
+      "13": 20,
+      "14": 21,
+      "15": 22,
+      "16": 23,
+      "17": 24,
+      "18": 25,
+      "19": 26,
+      "20": 27,
+      "21": 28,
+      "22": 29,
+      "23": 30,
+      "24": 31,
+      "25": 32,
+      "26": 33,
+      "27": 34,
+      "28": 35,
+      "29": 36,
+      "30": 37,
+      "31": 38,
+      "32": 39,
+      "33": 40,
+      "34": 41,
+      "35": 42,
+      "36": 43,
+      "37": 44,
+      "38": 45,
+      "39": 46,
+      "40": 47,
+      "41": 48,
+      "42": 49,
+      "43": 50,
+      "44": 51,
+      "45": 52,
+      "46": 53,
+      "47": 54,
+      "48": 55,
+      "49": 56,
+      "50": 57,
+      "51": 58,
+      "52": 59,
+      "53": 60,
+      "54": 61,
+      "55": 62,
+      "56": 63,
+      "57": 64,
+      "58": 65,
+      "59": 66,
+      "60": 67,
+      "61": 68,
+      "62": 69,
+      "63": 70,
+      "64": 71,
+      "65": 72,
+      "66": 73,
+      "67": 74,
+      "68": 75,
+      "69": 76,
+      "70": 77,
+      "71": 78,
+      "72": 79,
+      "73": 80,
+      "74": 81,
+      "75": 82,
+      "76": 83,
+      "77": 84,
+      "78": 85,
+      "79": 86,
+      "80": 87,
+      "81": 88,
+      "82": 89,
+      "83": 90,
+      "84": 91,
+      "85": 92,
+      "86": 93,
+      "87": 94,
+      "88": 95,
+      "89": 96,
+      "90": 97,
+      "91": 98,
+      "92": 99,
+      "93": 100,
+      "94": 101,
+      "95": 102,
+      "96": 103,
+      "97": 104,
+      "98": 105,
+      "99": 106
+    },
+    "merges": [
+      [
+        "1",
+        "0"
+      ],
+      [
+        "1",
+        "1"
+      ],
+      [
+        "1",
+        "2"
+      ],
+      [
+        "1",
+        "3"
+      ],
+      [
+        "1",
+        "4"
+      ],
+      [
+        "1",
+        "5"
+      ],
+      [
+        "1",
+        "6"
+      ],
+      [
+        "1",
+        "7"
+      ],
+      [
+        "1",
+        "8"
+      ],
+      [
+        "1",
+        "9"
+      ],
+      [
+        "2",
+        "0"
+      ],
+      [
+        "2",
+        "1"
+      ],
+      [
+        "2",
+        "2"
+      ],
+      [
+        "2",
+        "3"
+      ],
+      [
+        "2",
+        "4"
+      ],
+      [
+        "2",
+        "5"
+      ],
+      [
+        "2",
+        "6"
+      ],
+      [
+        "2",
+        "7"
+      ],
+      [
+        "2",
+        "8"
+      ],
+      [
+        "2",
+        "9"
+      ],
+      [
+        "3",
+        "0"
+      ],
+      [
+        "3",
+        "1"
+      ],
+      [
+        "3",
+        "2"
+      ],
+      [
+        "3",
+        "3"
+      ],
+      [
+        "3",
+        "4"
+      ],
+      [
+        "3",
+        "5"
+      ],
+      [
+        "3",
+        "6"
+      ],
+      [
+        "3",
+        "7"
+      ],
+      [
+        "3",
+        "8"
+      ],
+      [
+        "3",
+        "9"
+      ],
+      [
+        "4",
+        "0"
+      ],
+      [
+        "4",
+        "1"
+      ],
+      [
+        "4",
+        "2"
+      ],
+      [
+        "4",
+        "3"
+      ],
+      [
+        "4",
+        "4"
+      ],
+      [
+        "4",
+        "5"
+      ],
+      [
+        "4",
+        "6"
+      ],
+      [
+        "4",
+        "7"
+      ],
+      [
+        "4",
+        "8"
+      ],
+      [
+        "4",
+        "9"
+      ],
+      [
+        "5",
+        "0"
+      ],
+      [
+        "5",
+        "1"
+      ],
+      [
+        "5",
+        "2"
+      ],
+      [
+        "5",
+        "3"
+      ],
+      [
+        "5",
+        "4"
+      ],
+      [
+        "5",
+        "5"
+      ],
+      [
+        "5",
+        "6"
+      ],
+      [
+        "5",
+        "7"
+      ],
+      [
+        "5",
+        "8"
+      ],
+      [
+        "5",
+        "9"
+      ],
+      [
+        "6",
+        "0"
+      ],
+      [
+        "6",
+        "1"
+      ],
+      [
+        "6",
+        "2"
+      ],
+      [
+        "6",
+        "3"
+      ],
+      [
+        "6",
+        "4"
+      ],
+      [
+        "6",
+        "5"
+      ],
+      [
+        "6",
+        "6"
+      ],
+      [
+        "6",
+        "7"
+      ],
+      [
+        "6",
+        "8"
+      ],
+      [
+        "6",
+        "9"
+      ],
+      [
+        "7",
+        "0"
+      ],
+      [
+        "7",
+        "1"
+      ],
+      [
+        "7",
+        "2"
+      ],
+      [
+        "7",
+        "3"
+      ],
+      [
+        "7",
+        "4"
+      ],
+      [
+        "7",
+        "5"
+      ],
+      [
+        "7",
+        "6"
+      ],
+      [
+        "7",
+        "7"
+      ],
+      [
+        "7",
+        "8"
+      ],
+      [
+        "7",
+        "9"
+      ],
+      [
+        "8",
+        "0"
+      ],
+      [
+        "8",
+        "1"
+      ],
+      [
+        "8",
+        "2"
+      ],
+      [
+        "8",
+        "3"
+      ],
+      [
+        "8",
+        "4"
+      ],
+      [
+        "8",
+        "5"
+      ],
+      [
+        "8",
+        "6"
+      ],
+      [
+        "8",
+        "7"
+      ],
+      [
+        "8",
+        "8"
+      ],
+      [
+        "8",
+        "9"
+      ],
+      [
+        "9",
+        "0"
+      ],
+      [
+        "9",
+        "1"
+      ],
+      [
+        "9",
+        "2"
+      ],
+      [
+        "9",
+        "3"
+      ],
+      [
+        "9",
+        "4"
+      ],
+      [
+        "9",
+        "5"
+      ],
+      [
+        "9",
+        "6"
+      ],
+      [
+        "9",
+        "7"
+      ],
+      [
+        "9",
+        "8"
+      ],
+      [
+        "9",
+        "9"
+      ]
+    ]
+  }
+}

tokenizer_config.json

@@ -1 +1,74 @@
-{"data": {"tokenizer": {"name": "huggingface", "path": "flexitok/mod-tokenizers-ltr_2digit"}}}
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "mod",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "5": {
+      "content": "=",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "6": {
+      "content": " ",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "mod",
+    "=",
+    " "
+  ],
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "extra_special_tokens": {},
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "tokenizer_class": "PreTrainedTokenizerFast",
+  "unk_token": "<unk>",
+  "number_handling": "ltr_2digit"
+}

vocab.json

@@ -1,109 +1,109 @@
 {
-  "56": 63,
+  "22": 29,
+  "17": 24,
+  "14": 21,
+  "86": 93,
+  "68": 75,
+  "13": 20,
+  " ": 6,
+  "11": 18,
+  "<s>": 1,
+  "77": 84,
+  "<pad>": 3,
+  "24": 31,
+  "5": 12,
+  "=": 5,
+  "63": 70,
+  "61": 68,
+  "96": 103,
+  "98": 105,
   "93": 100,
-  "30": 37,
-  "51": 58,
-  "0": 7,
+  "55": 62,
+  "91": 98,
+  "3": 10,
+  "21": 28,
+  "40": 47,
+  "78": 85,
+  "59": 66,
+  "23": 30,
+  "52": 59,
+  "87": 94,
+  "64": 71,
+  "16": 23,
+  "8": 15,
+  "90": 97,
+  "49": 56,
+  "48": 55,
+  "</s>": 2,
+  "28": 35,
+  "60": 67,
+  "80": 87,
+  "34": 41,
+  "54": 61,
+  "35": 42,
+  "45": 52,
+  "4": 11,
   "79": 86,
-  "96": 103,
-  "37": 44,
-  "92": 99,
-  "95": 102,
-  "7": 14,
-  "32": 39,
-  "41": 48,
   "50": 57,
-  "71": 78,
-  "85": 92,
-  "59": 66,
-  "25": 32,
-  "11": 18,
-  "99": 106,
+  "58": 65,
+  "18": 25,
   "26": 33,
-  "88": 95,
-  "1": 8,
-  "74": 81,
-  "84": 91,
-  "46": 53,
   "83": 90,
-  "64": 71,
-  "21": 28,
-  "=": 5,
-  "69": 76,
-  "75": 82,
-  "44": 51,
-  "mod": 4,
-  "40": 47,
-  "68": 75,
-  "42": 49,
-  "28": 35,
-  "94": 101,
-  "90": 97,
+  "0": 7,
+  "19": 26,
+  "53": 60,
+  "25": 32,
+  "56": 63,
+  "62": 69,
+  "89": 96,
+  "29": 36,
   "65": 72,
   "70": 77,
   "66": 73,
-  "22": 29,
-  "98": 105,
-  "86": 93,
-  "33": 40,
+  "67": 74,
+  "6": 13,
+  "94": 101,
   "<unk>": 0,
-  "24": 31,
-  "43": 50,
-  "76": 83,
-  "38": 45,
-  "17": 24,
+  "33": 40,
+  "44": 51,
+  "12": 19,
+  "36": 43,
+  "75": 82,
   "82": 89,
-  "47": 54,
-  "61": 68,
-  "18": 25,
-  "73": 80,
-  "34": 41,
-  "67": 74,
-  "97": 104,
-  "<s>": 1,
-  "49": 56,
+  "2": 9,
+  "72": 79,
+  "76": 83,
+  "84": 91,
   "39": 46,
-  "78": 85,
-  "29": 36,
-  "89": 96,
-  "55": 62,
-  "6": 13,
-  "52": 59,
-  "45": 52,
-  "3": 10,
-  "4": 11,
-  "54": 61,
-  "<pad>": 3,
-  "53": 60,
-  "19": 26,
+  "51": 58,
+  "43": 50,
+  "71": 78,
   "10": 17,
-  "58": 65,
+  "74": 81,
+  "95": 102,
+  "81": 88,
+  "92": 99,
   "27": 34,
-  "91": 98,
-  "14": 21,
-  "23": 30,
-  "48": 55,
-  "62": 69,
-  " ": 6,
-  "35": 42,
+  "1": 8,
+  "30": 37,
+  "69": 76,
+  "73": 80,
   "31": 38,
-  "2": 9,
-  "77": 84,
-  "72": 79,
-  "36": 43,
-  "60": 67,
-  "80": 87,
-  "16": 23,
+  "7": 14,
+  "88": 95,
   "9": 16,
-  "</s>": 2,
-  "63": 70,
-  "8": 15,
-  "5": 12,
-  "81": 88,
-  "13": 20,
-  "57": 64,
-  "87": 94,
-  "20": 27,
+  "47": 54,
+  "99": 106,
+  "mod": 4,
+  "46": 53,
   "15": 22,
-  "12": 19
+  "97": 104,
+  "41": 48,
+  "20": 27,
+  "42": 49,
+  "32": 39,
+  "37": 44,
+  "57": 64,
+  "85": 92,
+  "38": 45
 }