import re
import collections
from transformers.tokenization_utils import PreTrainedTokenizer

VOCAB_FILES_NAMES = {'vocab_file': 'vocab.txt'}

def load_vocab(vocab_file):
    vocab = collections.OrderedDict()
    with open(vocab_file, "r", encoding="utf-8") as reader:
        tokens = reader.readlines()
    for index, token in enumerate(tokens):
        token = token.rstrip("\n")
        vocab[token] = index
    return vocab

"""
"""
class CharacterTokenizer(PreTrainedTokenizer):
    vocab_files_names = VOCAB_FILES_NAMES
    
    def __init__(self,
                 vocab_file,
                 model_max_length=2048,
                 add_prefix_space=False,
                 **kwargs):

        """Character tokenizer for Hugging Face transformers.
        """
        self.model_max_length = model_max_length        
        self._vocab_str_to_int = load_vocab(vocab_file)
        self._vocab_int_to_str = {v: k for k, v in self._vocab_str_to_int.items()}
        super().__init__(
            add_prefix_space=add_prefix_space,
            model_max_length=model_max_length,
            **kwargs,
        )

    @property
    def vocab_size(self):
        return len(self._vocab_str_to_int)

    def get_vocab(self):
        return self._vocab_str_to_int

    def _tokenize(self, text):
        return list(text)

    def _convert_token_to_id(self, token):
        return self._vocab_str_to_int.get(token, self._vocab_str_to_int["[UNK]"])

    def _convert_id_to_token(self, index):
        return self._vocab_int_to_str[index]

    def convert_tokens_to_string(self, tokens):
        return "".join(tokens)

    def build_inputs_with_special_tokens(
        self, token_ids_0, token_ids_1=None
    ):
        eos = [self.eos_token_id]
        sep = [self.sep_token_id]
        if token_ids_1 is None:
            result = token_ids_0 + eos
        else:
            result = token_ids_0 + eos + sep + token_ids_1 + eos
        return result