FlowRead / test_model2.py
jobbler's picture
Fix tokenizer extra_special_tokens crash and stabilize transformers
bd01d05
raw
history blame contribute delete
276 Bytes
from transformers import AutoTokenizer
try:
print("Loading Tokenizer with empty dict override...")
AutoTokenizer.from_pretrained("unsloth/gemma-4-E2B", extra_special_tokens={})
print("Loaded!")
except Exception as e:
import traceback
traceback.print_exc()