File size: 1,952 Bytes
42980a5 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 | import re
import collections
from transformers.tokenization_utils import PreTrainedTokenizer
VOCAB_FILES_NAMES = {'vocab_file': 'vocab.txt'}
def load_vocab(vocab_file):
vocab = collections.OrderedDict()
with open(vocab_file, "r", encoding="utf-8") as reader:
tokens = reader.readlines()
for index, token in enumerate(tokens):
token = token.rstrip("\n")
vocab[token] = index
return vocab
"""
"""
class CharacterTokenizer(PreTrainedTokenizer):
vocab_files_names = VOCAB_FILES_NAMES
def __init__(self,
vocab_file,
model_max_length=2048,
add_prefix_space=False,
**kwargs):
"""Character tokenizer for Hugging Face transformers.
"""
self.model_max_length = model_max_length
self._vocab_str_to_int = load_vocab(vocab_file)
self._vocab_int_to_str = {v: k for k, v in self._vocab_str_to_int.items()}
super().__init__(
add_prefix_space=add_prefix_space,
model_max_length=model_max_length,
**kwargs,
)
@property
def vocab_size(self):
return len(self._vocab_str_to_int)
def get_vocab(self):
return self._vocab_str_to_int
def _tokenize(self, text):
return list(text)
def _convert_token_to_id(self, token):
return self._vocab_str_to_int.get(token, self._vocab_str_to_int["[UNK]"])
def _convert_id_to_token(self, index):
return self._vocab_int_to_str[index]
def convert_tokens_to_string(self, tokens):
return "".join(tokens)
def build_inputs_with_special_tokens(
self, token_ids_0, token_ids_1=None
):
eos = [self.eos_token_id]
sep = [self.sep_token_id]
if token_ids_1 is None:
result = token_ids_0 + eos
else:
result = token_ids_0 + eos + sep + token_ids_1 + eos
return result
|