src/tokenizer/parse.py

from copy import deepcopy

from .spec import Tokenizer
from .tokenizer_part import TokenizerPart

def get_tokenizer(**kwargs) -> Tokenizer:
    __target__ = kwargs.get('__target__')
    assert __target__ is not None, "do not find `__target__` in tokenizer config"
    del kwargs['__target__']
    MAP = {
        'tokenizer_part': TokenizerPart,
    }
    return MAP[__target__].parse(**deepcopy(kwargs))