tokenization_olmo_fast.py

from hf_olmo.tokenization_olmo_fast import OLMoTokenizerFast, OLMoConfig
from transformers import AutoTokenizer

AutoTokenizer.register(OLMoConfig, fast_tokenizer_class=OLMoTokenizerFast)