load chat_template in from_pretrained (vLLM apply_chat_template fix)

tokenizer.py

@@ -580,29 +580,61 @@ class HybridDNATokenizer(PreTrainedTokenizer):
         return (save_directory,)
 
     @classmethod
-    def from_pretrained(cls, pretrained_model_name_or_path, *args, **kwargs):
+    def from_pretrained(
+        cls,
+        pretrained_model_name_or_path,
+        *args,
+        cache_dir=None,
+        local_files_only=False,
+        token=None,
+        revision="main",
+        trust_remote_code=False,
+        **kwargs,
+    ):
         # Custom loader: PreTrainedTokenizer.from_pretrained is incompatible with this
         # tokenizer's __init__ (it re-passes bos_token etc.), so construct directly from
         # bio_config/dna_config. That bypasses standard metadata loading, so also read
         # chat_template from tokenizer_config.json (vLLM / apply_chat_template need it).
-        import os as _os, json as _json
+        from transformers.utils.hub import cached_file
+
+        proxies = kwargs.pop("proxies", None)
+        subfolder = kwargs.pop("subfolder", None)
+        commit_hash = kwargs.pop("_commit_hash", None)
+        hub_kwargs = {
+            "cache_dir": cache_dir,
+            "local_files_only": local_files_only,
+            "proxies": proxies,
+            "revision": revision,
+            "subfolder": subfolder,
+            "token": token,
+            "_commit_hash": commit_hash,
+            "_raise_exceptions_for_missing_entries": False,
+        }
+
         cfg = {}
         for _name in ("bio_config.json", "dna_config.json"):
-            _p = _os.path.join(pretrained_model_name_or_path, _name)
-            if _os.path.exists(_p):
-                with open(_p, "r", encoding="utf-8") as _f:
-                    cfg = _json.load(_f)
+            _p = cached_file(pretrained_model_name_or_path, _name, **hub_kwargs)
+            if _p is not None:
+                with open(_p, encoding="utf-8") as _f:
+                    cfg = json.load(_f)
                 break
+
         _init = {"base_tokenizer_path": pretrained_model_name_or_path}
         for _key in ("k", "tail", "auto_dna_tags"):
             if _key in cfg:
                 _init[_key] = cfg[_key]
-        tok = cls(**_init, **kwargs)
-        if getattr(tok, "chat_template", None) is None:
-            _tc = _os.path.join(pretrained_model_name_or_path, "tokenizer_config.json")
-            if _os.path.exists(_tc):
-                with open(_tc, "r", encoding="utf-8") as _f:
-                    _c = _json.load(_f)
-                if _c.get("chat_template"):
-                    tok.chat_template = _c["chat_template"]
-        return tok
+
+        tokenizer_config_path = cached_file(pretrained_model_name_or_path, "tokenizer_config.json", **hub_kwargs)
+        if tokenizer_config_path is not None:
+            with open(tokenizer_config_path, encoding="utf-8") as _f:
+                tokenizer_config = json.load(_f)
+            if tokenizer_config.get("chat_template"):
+                _init["chat_template"] = tokenizer_config["chat_template"]
+
+        chat_template_path = cached_file(pretrained_model_name_or_path, "chat_template.jinja", **hub_kwargs)
+        if chat_template_path is not None:
+            with open(chat_template_path, encoding="utf-8") as _f:
+                _init["chat_template"] = _f.read()
+
+        _init.update(kwargs)
+        return cls(**_init)