fix: re-ordering special tokens

tokenization_arcade100k.py

@@ -42,12 +42,14 @@ def _arcade100k(vocab_file: str):
     mergeable_ranks = _load_tiktoken_bpe(vocab_file)
 
     ENDOFTEXT = "<|endoftext|>"
-    # StarCoder special tokens (https://huggingface.co/bigcode/starcoder/blob/main/tokenizer_config.json)
+    FIM = [
+        "<|fim_prefix|>",
+        "<|fim_middle|>",
+        "<|fim_suffix|>",
+        "<|fim_pad|>",
+    ]
+    # `StarCoder` Tokens
     CODE = [
-        "<fim_prefix>",
-        "<fim_middle>",
-        "<fim_suffix>",
-        "<fim_pad>",
         "<gh_stars>",
         "<filename>",
         "<issue_start>",
@@ -68,10 +70,9 @@ def _arcade100k(vocab_file: str):
         "<|im_end|>",   # Chat: Input message end
     ]
     PAUSE = "<|pause|>"       # Think before you speak (https://arxiv.org/abs/2310.02226)
-    REGISTERS = [f"<|reg{i}|>" for i in range(0, 8)]  # Register/sink tokens (https://arxiv.org/abs/2309.17453)
+    REGISTERS = [f"<|reg{i}|>" for i in range(0, 8)]  # Register 0 sink token (https://arxiv.org/abs/2309.17453)
     ENDOFPROMPT = "<|endofprompt|>"
-    
-    SPECIAL_TOKENS_NAMES = [ENDOFTEXT] + CODE + [ENDOFPROMPT] + CHAT + [PAUSE] + REGISTERS
+    SPECIAL_TOKENS_NAMES = [ENDOFTEXT] + FIM + CODE + [ENDOFPROMPT] + CHAT + [PAUSE] + REGISTERS + ["<|extra0|>"]
     START_ID = len(mergeable_ranks) + 1
     SPECIAL_TOKENS = {
         t: START_ID + i
@@ -110,8 +111,9 @@ class Arcade100kTokenizer(PreTrainedTokenizer):
         **kwargs,
     ):
         super().__init__(errors=errors, **kwargs)
-        self._tiktoken_config = _arcade100k(self.vocab_files_names["vocab_file"])
+        self._tiktoken_config = _arcade100k(vocab_file)
         self.tokenizer = tiktoken.Encoding(**self._tiktoken_config)
+
         # TODO: Remove this assertion
         assert (
             len(self.tokenizer._mergeable_ranks)
@@ -174,7 +176,7 @@ class Arcade100kTokenizer(PreTrainedTokenizer):
         Returns:
             `Tuple(str)`: Paths to the files saved.
         """
-        file_path = os.path.join(save_directory, "qwen.tiktoken")
+        file_path = os.path.join(save_directory, "arcade100k.tiktoken")
         with open(file_path, "w", encoding="utf8") as w:
             for k, v in self.tokenizer._mergeable_ranks.items():
                 line = base64.b64encode(k).decode("utf8") + " " + str(v) + "\n"