Update supertoken.py

supertoken.py

@@ -17,8 +17,6 @@ from typing import List, Optional
 import tokenizers
 import transformers
 import yaml
-from xarch_tokenizers.models import load_tokenizer as hf_load_tokenizer
-from xarch_tokenizers.utils import system
 
 Vocab = dict[str, list[int]]
 
@@ -112,12 +110,7 @@ class HFTokenizer(Tokenizer):
 
     @classmethod
     def load(cls, name):
-        if system.get_host() == system.Hosts.vector:
-            name = system.VECTOR_HF_MAPPING.get(name, name)
-        try:
-            tok = hf_load_tokenizer(name)
-        except:
-            tok = transformers.AutoTokenizer.from_pretrained(name)
+        tok = transformers.AutoTokenizer.from_pretrained(name)
         sts = getattr(tok, "special_tokens_map", {})
         if "bert" in name:
             bos_str = sts.get("cls_token")