src/processer/tokenizer.py

from transformers import BertTokenizer


def bert_tokenizer(pretrained_model_name_or_path):
    tokenizer = BertTokenizer.from_pretrained(pretrained_model_name_or_path=pretrained_model_name_or_path,
                                              truncation_side='right')
    tokenizer.add_special_tokens({"bos_token": "[DEC]"})
    return tokenizer