| import re |
| import collections |
| from transformers.tokenization_utils import PreTrainedTokenizer |
|
|
| VOCAB_FILES_NAMES = {'vocab_file': 'vocab.txt'} |
|
|
| def load_vocab(vocab_file): |
| vocab = collections.OrderedDict() |
| with open(vocab_file, "r", encoding="utf-8") as reader: |
| tokens = reader.readlines() |
| for index, token in enumerate(tokens): |
| token = token.rstrip("\n") |
| vocab[token] = index |
| return vocab |
|
|
| """ |
| """ |
| class CharacterTokenizer(PreTrainedTokenizer): |
| vocab_files_names = VOCAB_FILES_NAMES |
| |
| def __init__(self, |
| vocab_file, |
| model_max_length=2048, |
| add_prefix_space=False, |
| **kwargs): |
|
|
| """Character tokenizer for Hugging Face transformers. |
| """ |
| self.model_max_length = model_max_length |
| self._vocab_str_to_int = load_vocab(vocab_file) |
| self._vocab_int_to_str = {v: k for k, v in self._vocab_str_to_int.items()} |
| super().__init__( |
| add_prefix_space=add_prefix_space, |
| model_max_length=model_max_length, |
| **kwargs, |
| ) |
|
|
| @property |
| def vocab_size(self): |
| return len(self._vocab_str_to_int) |
|
|
| def get_vocab(self): |
| return self._vocab_str_to_int |
|
|
| def _tokenize(self, text): |
| return list(text) |
|
|
| def _convert_token_to_id(self, token): |
| return self._vocab_str_to_int.get(token, self._vocab_str_to_int["[UNK]"]) |
|
|
| def _convert_id_to_token(self, index): |
| return self._vocab_int_to_str[index] |
|
|
| def convert_tokens_to_string(self, tokens): |
| return "".join(tokens) |
|
|
| def build_inputs_with_special_tokens( |
| self, token_ids_0, token_ids_1=None |
| ): |
| eos = [self.eos_token_id] |
| sep = [self.sep_token_id] |
| if token_ids_1 is None: |
| result = token_ids_0 + eos |
| else: |
| result = token_ids_0 + eos + sep + token_ids_1 + eos |
| return result |
|
|
|
|
