| import os |
|
|
| import torch |
| from tokenizers import Tokenizer |
|
|
| from TTS.tts.utils.text.cleaners import english_cleaners |
|
|
| DEFAULT_VOCAB_FILE = os.path.join( |
| os.path.dirname(os.path.realpath(__file__)), "../../utils/assets/tortoise/tokenizer.json" |
| ) |
|
|
|
|
| class VoiceBpeTokenizer: |
| def __init__(self, vocab_file=DEFAULT_VOCAB_FILE, vocab_str=None): |
| self.tokenizer = None |
| if vocab_file is not None: |
| self.tokenizer = Tokenizer.from_file(vocab_file) |
| if vocab_str is not None: |
| self.tokenizer = Tokenizer.from_str(vocab_str) |
|
|
| def preprocess_text(self, txt): |
| txt = english_cleaners(txt) |
| return txt |
|
|
| def encode(self, txt): |
| txt = self.preprocess_text(txt) |
| txt = txt.replace(" ", "[SPACE]") |
| return self.tokenizer.encode(txt).ids |
|
|
| def decode(self, seq): |
| if isinstance(seq, torch.Tensor): |
| seq = seq.cpu().numpy() |
| txt = self.tokenizer.decode(seq, skip_special_tokens=False).replace(" ", "") |
| txt = txt.replace("[SPACE]", " ") |
| txt = txt.replace("[STOP]", "") |
| txt = txt.replace("[UNK]", "") |
| return txt |
|
|
