src/rag/tokenizer.py

import underthesea

def vi_tokenizer(text):
    return underthesea.word_tokenize(text.lower())