use_custom_tokenizer.py

from spacy.util import registry

from commercial_registry_ner.spacy.custom_tokenizer.custom_tokenizer import (
    custom_tokenizer,
)


@registry.tokenizers("customize_tokenizer")
def make_customize_tokenizer():
    def customize_tokenizer(nlp):
        return custom_tokenizer(nlp)

    return customize_tokenizer