fix: make `eos_token`/`pad_token` overridable

tokenization_arcade100k.py

@@ -124,8 +124,12 @@ class Arcade100kTokenizer(PreTrainedTokenizer):
 
         self.decoder = {i: n for n, i in self.tokenizer._mergeable_ranks.items()}
         self.decoder.update({i: n for n, i in self.tokenizer._special_tokens.items()})
-        self.eos_token = self.decoder[self.tokenizer.eot_token]
-        self.pad_token = self.decoder[self.tokenizer.eot_token]
+        # Provide default `eos_token` and `pad_token`
+        if self.eos_token is None:
+            self.eos_token = self.decoder[self.tokenizer.eot_token]
+        if self.pad_token is None:
+            self.pad_token = self.decoder[self.tokenizer.pad_token]
+
         # Expose for convenience
         self.mergeable_ranks = self.tokenizer._mergeable_ranks
         self.special_tokens = self.tokenizer._special_tokens

tokenizer_config.json

@@ -5,5 +5,7 @@
       "tokenization_arcade100k.Arcade100kTokenizer",
       null
     ]
-  }
+  },
+  "eos_token": "<|endoftext|>",
+  "pad_token": "<|endoftext|>"
 }