commit of fast smoltokenizer

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

added_tokens.json

@@ -1,10 +0,0 @@
-{
-  "<file_sep>": 264,
-  "<fim_middle>": 260,
-  "<fim_pad>": 262,
-  "<fim_prefix>": 259,
-  "<fim_suffix>": 261,
-  "<repo_name>": 263,
-  "<|im_end|>": 265,
-  "<|im_start|>": 266
-}

special_tokens_map.json

@@ -5,30 +5,7 @@
     "<fim_middle>",
     "<fim_suffix>",
     "<fim_pad>",
-    "<repo_name>",
-    "<file_sep>",
     "<|im_end|>",
     "<|im_start|>"
-  ],
-  "eos_token": {
-    "content": "<|endoftext|>",
-    "lstrip": true,
-    "normalized": true,
-    "rstrip": true,
-    "single_word": false
-  },
-  "pad_token": {
-    "content": "<pad>",
-    "lstrip": true,
-    "normalized": true,
-    "rstrip": true,
-    "single_word": false
-  },
-  "unk_token": {
-    "content": "<unk>",
-    "lstrip": true,
-    "normalized": true,
-    "rstrip": true,
-    "single_word": false
-  }
+  ]
 }

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a5078c05e94e5980118e913ed49426e7b1cdf93dcc6ccefa7967fafd2f7e255e
+size 6166

tokenizer_config.json

@@ -1,31 +1,7 @@
 {
   "added_tokens_decoder": {
-    "0": {
-      "content": "<pad>",
-      "lstrip": true,
-      "normalized": true,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
-    "1": {
-      "content": "<|endoftext|>",
-      "lstrip": true,
-      "normalized": true,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
-    "2": {
-      "content": "<unk>",
-      "lstrip": true,
-      "normalized": true,
-      "rstrip": true,
-      "single_word": false,
-      "special": true
-    },
     "259": {
-      "content": "<fim_prefix>",
+      "content": "<|endoftext|>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -33,7 +9,7 @@
       "special": true
     },
     "260": {
-      "content": "<fim_middle>",
+      "content": "<fim_prefix>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -41,7 +17,7 @@
       "special": true
     },
     "261": {
-      "content": "<fim_suffix>",
+      "content": "<fim_middle>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -49,7 +25,7 @@
       "special": true
     },
     "262": {
-      "content": "<fim_pad>",
+      "content": "<fim_suffix>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -57,7 +33,7 @@
       "special": true
     },
     "263": {
-      "content": "<repo_name>",
+      "content": "<fim_pad>",
       "lstrip": false,
       "normalized": false,
       "rstrip": false,
@@ -65,14 +41,6 @@
       "special": true
     },
     "264": {
-      "content": "<file_sep>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "265": {
       "content": "<|im_end|>",
       "lstrip": false,
       "normalized": false,
@@ -80,7 +48,7 @@
       "single_word": false,
       "special": true
     },
-    "266": {
+    "265": {
       "content": "<|im_start|>",
       "lstrip": false,
       "normalized": false,
@@ -95,16 +63,10 @@
     "<fim_middle>",
     "<fim_suffix>",
     "<fim_pad>",
-    "<repo_name>",
-    "<file_sep>",
     "<|im_end|>",
     "<|im_start|>"
   ],
   "clean_up_tokenization_spaces": true,
-  "eos_token": "<|endoftext|>",
-  "extra_ids": 0,
   "model_max_length": 1000000000000000019884624838656,
-  "pad_token": "<pad>",
-  "tokenizer_class": "ByT5Tokenizer",
-  "unk_token": "<unk>"
+  "tokenizer_class": "PreTrainedTokenizerFast"
 }