File size: 1,138 Bytes
942788e | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 | """Make tokenizer loading work with HF_HUB_OFFLINE=1 on transformers 4.57.x.
PreTrainedTokenizerBase._patch_mistral_regex calls huggingface_hub.model_info() for every hub-named tokenizer with a
vocab > 100k (to detect Mistral base models). With HF_HUB_OFFLINE=1 that raises OfflineModeIsEnabled even though all
files are in the cache. GR00T's Cosmos-Reason2-2B / Qwen3-VL tokenizer is not a Mistral tokenizer, so when offline the
check is skipped and the tokenizer is returned unchanged. Online behaviour is untouched.
"""
import os
from transformers.tokenization_utils_base import PreTrainedTokenizerBase as _PTB
if not getattr(_PTB, "_hb_offline_fix", False):
_orig = _PTB._patch_mistral_regex.__func__
def _patch_mistral_regex(cls, tokenizer, *args, **kwargs):
try:
return _orig(cls, tokenizer, *args, **kwargs)
except Exception as error:
if os.environ.get("HF_HUB_OFFLINE") == "1" or type(error).__name__ == "OfflineModeIsEnabled":
return tokenizer
raise
_PTB._patch_mistral_regex = classmethod(_patch_mistral_regex)
_PTB._hb_offline_fix = True
|