teddybear082 commited on
Commit Β·
d00ff23
1
Parent(s): a05aaa6
linting fixes
Browse files- app/services/preprocess.py +9 -11
app/services/preprocess.py
CHANGED
|
@@ -5,8 +5,6 @@ See license at: https://github.com/KittenML/KittenTTS/blob/main/LICENSE (Apache
|
|
| 5 |
|
| 6 |
import re
|
| 7 |
import unicodedata
|
| 8 |
-
from typing import Optional
|
| 9 |
-
|
| 10 |
|
| 11 |
# βββββββββββββββββββββββββββββββββββββββββββββ
|
| 12 |
# Number β Words conversion
|
|
@@ -91,7 +89,7 @@ def number_to_words(n: int) -> str:
|
|
| 91 |
return f"{_ONES[hundreds]} hundred"
|
| 92 |
|
| 93 |
parts = []
|
| 94 |
-
for
|
| 95 |
chunk = n % 1000
|
| 96 |
if chunk:
|
| 97 |
chunk_words = _three_digits_to_words(chunk)
|
|
@@ -187,7 +185,7 @@ _RE_PERCENT = re.compile(r"(-?[\d,]+(?:\.\d+)?)\s*%")
|
|
| 187 |
|
| 188 |
# Currency: $100, β¬1,200.50, Β£50, $85K, $2.5M (optional scale suffix)
|
| 189 |
_RE_CURRENCY = re.compile(
|
| 190 |
-
r"([$β¬Β£Β₯βΉβ©βΏ])\s*([\d,]+(?:\.\d+)?)\s*(million|billion|trillion|thousand|[KMBT])?\b",
|
| 191 |
re.IGNORECASE
|
| 192 |
)
|
| 193 |
|
|
@@ -245,7 +243,7 @@ def expand_symbols(text: str) -> str:
|
|
| 245 |
text = _RE_AMPERSAND.sub(" and ", text)
|
| 246 |
text = _RE_AT_SYMBOL.sub(" at ", text)
|
| 247 |
return text
|
| 248 |
-
|
| 249 |
def _ordinal_suffix(n: int) -> str:
|
| 250 |
"""Return the ordinal word for n (e.g. 1 β 'first', 5 β 'fifth', 21 β 'twenty-first')."""
|
| 251 |
word = number_to_words(n)
|
|
@@ -328,7 +326,7 @@ def expand_currency(text: str) -> str:
|
|
| 328 |
def _replace(m: re.Match) -> str:
|
| 329 |
symbol = m.group(1)
|
| 330 |
raw = m.group(2).replace(",", "")
|
| 331 |
-
scale_suffix = m.group(3)
|
| 332 |
unit = _CURRENCY_SYMBOLS.get(symbol, "")
|
| 333 |
|
| 334 |
# Handle Scaled Currency ($17.5 billion or $17.5B)
|
|
@@ -650,10 +648,10 @@ def expand_months(text: str) -> str:
|
|
| 650 |
|
| 651 |
# 1. Standard abbreviations
|
| 652 |
text = _RE_MONTHS.sub(_replace, text)
|
| 653 |
-
|
| 654 |
# 2. May (Special case: only if followed by a digit)
|
| 655 |
text = _RE_MAY.sub("May", text) # Essentially just ensuring it's treated as a word
|
| 656 |
-
|
| 657 |
return text
|
| 658 |
|
| 659 |
# βββββββββββββββββββββββββββββββββββββββββββββ
|
|
@@ -761,7 +759,7 @@ def expand_contractions(text: str) -> str:
|
|
| 761 |
return text
|
| 762 |
|
| 763 |
|
| 764 |
-
def remove_stopwords(text: str, stopwords:
|
| 765 |
"""
|
| 766 |
Remove stopwords from text.
|
| 767 |
|
|
@@ -833,7 +831,7 @@ class TextPreprocessor:
|
|
| 833 |
remove_mentions: bool = False,
|
| 834 |
remove_punctuation: bool = True,
|
| 835 |
remove_stopwords: bool = False,
|
| 836 |
-
stopwords:
|
| 837 |
normalize_unicode: bool = True,
|
| 838 |
remove_accents: bool = False,
|
| 839 |
remove_extra_whitespace: bool = True,
|
|
@@ -918,4 +916,4 @@ class TextPreprocessor:
|
|
| 918 |
if cfg["remove_extra_whitespace"]:
|
| 919 |
text = remove_extra_whitespace(text)
|
| 920 |
|
| 921 |
-
return text
|
|
|
|
| 5 |
|
| 6 |
import re
|
| 7 |
import unicodedata
|
|
|
|
|
|
|
| 8 |
|
| 9 |
# βββββββββββββββββββββββββββββββββββββββββββββ
|
| 10 |
# Number β Words conversion
|
|
|
|
| 89 |
return f"{_ONES[hundreds]} hundred"
|
| 90 |
|
| 91 |
parts = []
|
| 92 |
+
for _i, scale in enumerate(_SCALE):
|
| 93 |
chunk = n % 1000
|
| 94 |
if chunk:
|
| 95 |
chunk_words = _three_digits_to_words(chunk)
|
|
|
|
| 185 |
|
| 186 |
# Currency: $100, β¬1,200.50, Β£50, $85K, $2.5M (optional scale suffix)
|
| 187 |
_RE_CURRENCY = re.compile(
|
| 188 |
+
r"([$β¬Β£Β₯βΉβ©βΏ])\s*([\d,]+(?:\.\d+)?)\s*(million|billion|trillion|thousand|[KMBT])?\b",
|
| 189 |
re.IGNORECASE
|
| 190 |
)
|
| 191 |
|
|
|
|
| 243 |
text = _RE_AMPERSAND.sub(" and ", text)
|
| 244 |
text = _RE_AT_SYMBOL.sub(" at ", text)
|
| 245 |
return text
|
| 246 |
+
|
| 247 |
def _ordinal_suffix(n: int) -> str:
|
| 248 |
"""Return the ordinal word for n (e.g. 1 β 'first', 5 β 'fifth', 21 β 'twenty-first')."""
|
| 249 |
word = number_to_words(n)
|
|
|
|
| 326 |
def _replace(m: re.Match) -> str:
|
| 327 |
symbol = m.group(1)
|
| 328 |
raw = m.group(2).replace(",", "")
|
| 329 |
+
scale_suffix = m.group(3)
|
| 330 |
unit = _CURRENCY_SYMBOLS.get(symbol, "")
|
| 331 |
|
| 332 |
# Handle Scaled Currency ($17.5 billion or $17.5B)
|
|
|
|
| 648 |
|
| 649 |
# 1. Standard abbreviations
|
| 650 |
text = _RE_MONTHS.sub(_replace, text)
|
| 651 |
+
|
| 652 |
# 2. May (Special case: only if followed by a digit)
|
| 653 |
text = _RE_MAY.sub("May", text) # Essentially just ensuring it's treated as a word
|
| 654 |
+
|
| 655 |
return text
|
| 656 |
|
| 657 |
# βββββββββββββββββββββββββββββββββββββββββββββ
|
|
|
|
| 759 |
return text
|
| 760 |
|
| 761 |
|
| 762 |
+
def remove_stopwords(text: str, stopwords: set | None = None) -> str:
|
| 763 |
"""
|
| 764 |
Remove stopwords from text.
|
| 765 |
|
|
|
|
| 831 |
remove_mentions: bool = False,
|
| 832 |
remove_punctuation: bool = True,
|
| 833 |
remove_stopwords: bool = False,
|
| 834 |
+
stopwords: set | None = None,
|
| 835 |
normalize_unicode: bool = True,
|
| 836 |
remove_accents: bool = False,
|
| 837 |
remove_extra_whitespace: bool = True,
|
|
|
|
| 916 |
if cfg["remove_extra_whitespace"]:
|
| 917 |
text = remove_extra_whitespace(text)
|
| 918 |
|
| 919 |
+
return text
|