FlowRead / test_bos.py
jobbler's picture
Fix tokenizer extra_special_tokens crash and stabilize transformers
bd01d05
raw
history blame contribute delete
228 Bytes
from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained("unsloth/gemma-4-E2B", extra_special_tokens={})
print("add_bos_token:", tokenizer.add_bos_token)
print("tokens:", tokenizer("hello")["input_ids"])