File size: 187 Bytes
5fc231f
 
 
 
1
2
3
4
from hf_olmo.tokenization_olmo_fast import OLMoTokenizerFast, OLMoConfig
from transformers import AutoTokenizer

AutoTokenizer.register(OLMoConfig, fast_tokenizer_class=OLMoTokenizerFast)