support older transformer versions

tokenization_qwen.py

@@ -18,7 +18,7 @@ from PIL import Image
 from PIL import ImageFont
 from PIL import ImageDraw
 from transformers import PreTrainedTokenizer, AddedToken
-from transformers.utils import try_to_load_from_cache
+# from transformers.utils import try_to_load_from_cache
 
 import matplotlib.colors as mcolors
 from matplotlib.font_manager import FontProperties
@@ -27,12 +27,12 @@ logger = logging.getLogger(__name__)
 
 
 VOCAB_FILES_NAMES = {"vocab_file": "qwen.tiktoken", "ttf": "SimSun.ttf"}
-FONT_PATH = try_to_load_from_cache("Qwen/Qwen-VL-Chat", "SimSun.ttf")
-if FONT_PATH is None:
-    if not os.path.exists("SimSun.ttf"):
-        ttf = requests.get("https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen-VL/assets/SimSun.ttf")
-        open("SimSun.ttf", "wb").write(ttf.content)
-    FONT_PATH = "SimSun.ttf"
+# FONT_PATH = try_to_load_from_cache("Qwen/Qwen-VL-Chat", "SimSun.ttf")
+# if FONT_PATH is None:
+#     if not os.path.exists("SimSun.ttf"):
+#         ttf = requests.get("https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen-VL/assets/SimSun.ttf")
+#         open("SimSun.ttf", "wb").write(ttf.content)
+#     FONT_PATH = "SimSun.ttf"
 
 PAT_STR = r"""(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+"""
 ENDOFTEXT = "<|endoftext|>"