5fc231f
1
2
3
4
from hf_olmo.tokenization_olmo_fast import OLMoTokenizerFast, OLMoConfig from transformers import AutoTokenizer AutoTokenizer.register(OLMoConfig, fast_tokenizer_class=OLMoTokenizerFast)
