| import os |
| import json |
|
|
| _special_tokens_phonemes = list('!"(),./:;?[]{}¡ª°´º¿\'') |
| _special_tokens_graphemes = ['a', 'á', 'e', 'é', 'i', 'í', 'o', 'ó', 'u', 'ú', 'b', 'c', 'ch', 'd', 'f', 'g', 'gu', |
| 'h', 'j', 'k', 'l', 'll', 'm', 'n', 'nh', 'ñ', 'p', 'q', 'qu', 'r', 'rr', 's', 't', 'v', 'x', 'z'] |
|
|
| |
|
|
| _module_dir = os.path.dirname(__file__) |
| _json_path = os.path.join(_module_dir, 'phoneme_token_maps.json') |
|
|
| symbols_phonemes = [] |
| symbols_graphemes = [] |
|
|
| if os.path.exists(_json_path): |
| try: |
| with open(_json_path, 'r', encoding='utf-8') as fh: |
| j = json.load(fh) |
|
|
| for k, v in j.items(): |
| phoneme = v.get('phoneme') |
| grapheme = v.get('grapheme') |
| if phoneme not in symbols_phonemes: |
| symbols_phonemes.append(phoneme) |
| if grapheme not in symbols_graphemes: |
| symbols_graphemes.append(grapheme) |
| for special_token in _special_tokens_phonemes: |
| if special_token not in symbols_phonemes: |
| symbols_phonemes.append(special_token) |
| for special_token in _special_tokens_graphemes: |
| if special_token not in symbols_graphemes: |
| symbols_graphemes.append(special_token) |
|
|
| except Exception as e: |
| print(f"Error loading phoneme_token_maps.json: {e}") |
|
|
| |
| symbol_to_id_phonemes = {symbol: i for i, |
| symbol in enumerate(symbols_phonemes)} |
| id_to_symbol_phonemes = {i: symbol for i, |
| symbol in enumerate(symbols_phonemes)} |
|
|
| symbol_to_id_graphemes = {symbol: i for i, |
| symbol in enumerate(symbols_graphemes)} |
| id_to_symbol_graphemes = {i: symbol for i, |
| symbol in enumerate(symbols_graphemes)} |
|
|
| print("Phoneme symbols:\n") |
| for i, s in enumerate(symbols_phonemes): |
| print(f"{i:3d}: {s}") |
| print("Grapheme symbols:\n") |
| for i, s in enumerate(symbols_graphemes): |
| print(f"{i:3d}: {s}") |
|
|
|
|
| TOKEN_SIZE = len(symbols_phonemes) |
| VOCAB_SIZE = len(symbols_graphemes) |
|
|
|
|
| class TextCleanerGal: |
| def __init__(self): |
| self.phoneme_to_id = symbol_to_id_phonemes |
| self.grapheme_to_id = symbol_to_id_graphemes |
|
|
| self.id_to_phoneme = id_to_symbol_phonemes |
| self.id_to_grapheme = id_to_symbol_graphemes |
|
|
| def __call__(self, tokens, mode="phoneme"): |
| if mode == "phoneme": |
| dictionary = self.phoneme_to_id |
| unk = "X" |
| else: |
| dictionary = self.grapheme_to_id |
| unk = " " |
| return [dictionary.get(tok, dictionary[unk]) for tok in tokens] |
|
|
| def decode_phoneme(self, idxs): |
| return "".join(self.id_to_phoneme.get(i, "X") for i in idxs) |
|
|
| def decode_grapheme(self, idxs): |
| return "".join(self.id_to_grapheme.get(i, "X") for i in idxs) |
|
|
