File size: 829 Bytes
47dfda4 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 | import json
class CharTokenizer:
def __init__(self, vocab=None):
if vocab is None:
self.itos = []
else:
self.itos = list(vocab)
self.stoi = {ch: i for i, ch in enumerate(self.itos)}
@classmethod
def from_text(cls, text):
return cls(sorted(set(text)))
@property
def vocab_size(self):
return len(self.itos)
def encode(self, text):
return [self.stoi[ch] for ch in text if ch in self.stoi]
def decode(self, ids):
return "".join(self.itos[i] for i in ids)
def save(self, path):
with open(path, "w", encoding="utf-8") as f:
json.dump(self.itos, f, ensure_ascii=False)
@classmethod
def load(cls, path):
with open(path, encoding="utf-8") as f:
return cls(json.load(f))
|