teddybear082 commited on
Commit
ec11370
Β·
1 Parent(s): d00ff23

more linting

Browse files
Files changed (2) hide show
  1. app/routes.py +2 -2
  2. app/services/preprocess.py +399 -225
app/routes.py CHANGED
@@ -164,9 +164,9 @@ def generate_speech():
164
  use_text_preprocess = current_app.config.get('TEXT_PREPROCESS_DEFAULT', False)
165
  # Preprocess text
166
  if use_text_preprocess:
167
- #logger.info(f'Preprocessing text: {text}')
168
  text = text_preprocessor.process(text)
169
- #logger.info(f'Preprocessed text: {text}')
170
  if use_streaming:
171
  return _stream_audio(tts, voice_state, text, target_format)
172
  return _generate_file(tts, voice_state, text, target_format)
 
164
  use_text_preprocess = current_app.config.get('TEXT_PREPROCESS_DEFAULT', False)
165
  # Preprocess text
166
  if use_text_preprocess:
167
+ # logger.info(f'Preprocessing text: {text}')
168
  text = text_preprocessor.process(text)
169
+ # logger.info(f'Preprocessed text: {text}')
170
  if use_streaming:
171
  return _stream_audio(tts, voice_state, text, target_format)
172
  return _generate_file(tts, voice_state, text, target_format)
app/services/preprocess.py CHANGED
@@ -11,56 +11,99 @@ import unicodedata
11
  # ─────────────────────────────────────────────
12
 
13
  _ONES = [
14
- "", "one", "two", "three", "four", "five", "six", "seven", "eight", "nine",
15
- "ten", "eleven", "twelve", "thirteen", "fourteen", "fifteen", "sixteen",
16
- "seventeen", "eighteen", "nineteen",
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
17
  ]
18
- _TENS = ["", "", "twenty", "thirty", "forty", "fifty", "sixty", "seventy", "eighty", "ninety"]
19
- _SCALE = ["", "thousand", "million", "billion", "trillion"]
20
 
21
  _ORDINAL_EXCEPTIONS = {
22
- "one": "first", "two": "second", "three": "third", "four": "fourth",
23
- "five": "fifth", "six": "sixth", "seven": "seventh", "eight": "eighth",
24
- "nine": "ninth", "twelve": "twelfth",
 
 
 
 
 
 
 
25
  }
26
 
27
  _CURRENCY_SYMBOLS = {
28
- "$": "dollar", "€": "euro", "Β£": "pound", "Β₯": "yen",
29
- "β‚Ή": "rupee", "β‚©": "won", "β‚Ώ": "bitcoin",
 
 
 
 
 
30
  }
31
 
32
  _CURRENCY_SCALE_MAP = {
33
- "K": "thousand", "M": "million", "B": "billion", "T": "trillion",
34
- "thousand": "thousand", "million": "million", "billion": "billion", "trillion": "trillion"
 
 
 
 
 
 
35
  }
36
 
37
  _ROMAN = [
38
- (1000, "M"), (900, "CM"), (500, "D"), (400, "CD"),
39
- (100, "C"), (90, "XC"), (50, "L"), (40, "XL"),
40
- (10, "X"), (9, "IX"), (5, "V"), (4, "IV"), (1, "I"),
 
 
 
 
 
 
 
 
 
 
41
  ]
42
- _RE_ROMAN = re.compile(
43
- r"\b(M{0,4})(CM|CD|D?C{0,3})(XC|XL|L?X{0,3})(IX|IV|V?I{0,3})\b"
44
- )
45
 
46
 
47
  def _three_digits_to_words(n: int) -> str:
48
  """Convert a number 0–999 to English words."""
49
  if n == 0:
50
- return ""
51
  parts = []
52
  hundreds = n // 100
53
  remainder = n % 100
54
  if hundreds:
55
- parts.append(f"{_ONES[hundreds]} hundred")
56
  if remainder < 20:
57
  if remainder:
58
  parts.append(_ONES[remainder])
59
  else:
60
  tens_word = _TENS[remainder // 10]
61
  ones_word = _ONES[remainder % 10]
62
- parts.append(f"{tens_word}-{ones_word}" if ones_word else tens_word)
63
- return " ".join(parts)
64
 
65
 
66
  def number_to_words(n: int) -> str:
@@ -77,31 +120,31 @@ def number_to_words(n: int) -> str:
77
  if not isinstance(n, int):
78
  n = int(n)
79
  if n == 0:
80
- return "zero"
81
  if n < 0:
82
- return f"negative {number_to_words(-n)}"
83
 
84
  # X00–X999 read as "X hundred" (e.g. 1200 β†’ "twelve hundred")
85
  # Exclude exact multiples of 1000 (1000 β†’ "one thousand", not "ten hundred")
86
  if 100 <= n <= 9999 and n % 100 == 0 and n % 1000 != 0:
87
  hundreds = n // 100
88
  if hundreds < 20:
89
- return f"{_ONES[hundreds]} hundred"
90
 
91
  parts = []
92
  for _i, scale in enumerate(_SCALE):
93
  chunk = n % 1000
94
  if chunk:
95
  chunk_words = _three_digits_to_words(chunk)
96
- parts.append(f"{chunk_words} {scale}".strip() if scale else chunk_words)
97
  n //= 1000
98
  if n == 0:
99
  break
100
 
101
- return " ".join(reversed(parts))
102
 
103
 
104
- def float_to_words(value, decimal_sep: str = "point") -> str:
105
  """
106
  Convert a float (or numeric string) to words, reading decimal digits individually.
107
  Accepts a string to preserve trailing zeros (e.g. "1.50" β†’ "one point five zero").
@@ -112,28 +155,27 @@ def float_to_words(value, decimal_sep: str = "point") -> str:
112
  "3.10" β†’ "three point one zero"
113
  1.007 β†’ "one point zero zero seven"
114
  """
115
- text = value if isinstance(value, str) else f"{value}"
116
- negative = text.startswith("-")
117
  if negative:
118
  text = text[1:]
119
 
120
- if "." in text:
121
- int_part, dec_part = text.split(".", 1)
122
- int_words = number_to_words(int(int_part)) if int_part else "zero"
123
  # Read each decimal digit individually; "0" β†’ "zero"
124
- digit_map = ["zero"] + _ONES[1:] # index 0 β†’ "zero"
125
- dec_words = " ".join(digit_map[int(d)] for d in dec_part)
126
- result = f"{int_words} {decimal_sep} {dec_words}"
127
  else:
128
  result = number_to_words(int(text))
129
 
130
- return f"negative {result}" if negative else result
131
 
132
 
133
  def roman_to_int(s: str) -> int:
134
  """Convert a Roman numeral string to an integer."""
135
- val = {"I": 1, "V": 5, "X": 10, "L": 50,
136
- "C": 100, "D": 500, "M": 1000}
137
  result = 0
138
  prev = 0
139
  for ch in reversed(s.upper()):
@@ -147,77 +189,88 @@ def roman_to_int(s: str) -> int:
147
  # Regex patterns
148
  # ─────────────────────────────────────────────
149
 
150
- _RE_URL = re.compile(r"https?://\S+|www\.\S+")
151
- _RE_EMAIL = re.compile(r"\b[\w.+-]+@[\w-]+\.[a-z]{2,}\b", re.IGNORECASE)
152
- _RE_HASHTAG = re.compile(r"#\w+")
153
- _RE_MENTION = re.compile(r"@\w+")
154
- _RE_HTML = re.compile(r"<[^>]+>")
155
- _RE_PUNCT = re.compile(r"[^\w\s]")
156
- _RE_SPACES = re.compile(r"\s+")
157
- _RE_AI = re.compile(r"\bAI\b")
158
- _RE_DOT_COM = re.compile(r"\.com\b", re.IGNORECASE)
159
- _RE_PLUS = re.compile(r"\+")
160
- _RE_AMPERSAND = re.compile(r"&")
161
- _RE_AT_SYMBOL = re.compile(r"@")
162
- _RE_NEWLINE = re.compile(r"[\r\n]+")
163
- _RE_TILDE = re.compile(r"~")
164
 
165
  _MONTH_MAP = {
166
- "Jan": "January", "Feb": "February", "Mar": "March", "Apr": "April",
167
- "Jun": "June", "Jul": "July", "Aug": "August", "Sep": "September",
168
- "Sept": "September", "Oct": "October", "Nov": "November", "Dec": "December"
 
 
 
 
 
 
 
 
 
169
  }
170
 
171
  # Regex looks for Title Case months followed by a period or a digit
172
  # We handle "May" separately because it's a common word.
173
- _RE_MONTHS = re.compile(r"\b(Jan|Feb|Mar|Apr|Jun|Jul|Aug|Sep|Sept|Oct|Nov|Dec)\.?\b(?=\s*\d|\s*$)")
174
- _RE_MAY = re.compile(r"\bMay\b(?=\s*\d)") # Only expand May if followed by a number (May 5)
175
 
176
  # Number: do NOT match a leading minus if it is immediately preceded by a letter
177
  # (handles "gpt-3", "gpl-3", "v-2" etc.)
178
- _RE_NUMBER = re.compile(r"(?<![a-zA-Z])-?[\d,]+(?:\.\d+)?")
179
 
180
  # Ordinals: 1st, 2nd, 3rd, 4th … 21st, 101st …
181
- _RE_ORDINAL = re.compile(r"\b(\d+)(st|nd|rd|th)\b", re.IGNORECASE)
182
 
183
  # Percentages: 50%, 3.5%
184
- _RE_PERCENT = re.compile(r"(-?[\d,]+(?:\.\d+)?)\s*%")
185
 
186
  # Currency: $100, €1,200.50, Β£50, $85K, $2.5M (optional scale suffix)
187
  _RE_CURRENCY = re.compile(
188
- r"([$€£Β₯β‚Ήβ‚©β‚Ώ])\s*([\d,]+(?:\.\d+)?)\s*(million|billion|trillion|thousand|[KMBT])?\b",
189
- re.IGNORECASE
190
  )
191
 
192
  # Time: 3:30pm, 14:00, 3:30 AM β€” requires 2-digit minutes so "3:0" (score) doesn't match
193
- _RE_TIME = re.compile(r"\b(\d{1,2}):(\d{2})(?::(\d{2}))?\s*(am|pm)?\b", re.IGNORECASE)
194
 
195
  # Ranges: 10-20, 100-200 (both sides numeric, hyphen between them)
196
- _RE_RANGE = re.compile(r"(?<!\w)(\d+)-(\d+)(?!\w)")
197
 
198
  # Version/model names: gpt-3, gpt-3.5, v2.0, Python-3.10, GPL-3
199
  # Letter(s) + hyphen + digit(s) [+ more version parts]
200
- _RE_MODEL_VER = re.compile(r"\b([a-zA-Z][a-zA-Z0-9]*)-(\d[\d.]*)(?=[^\d.]|$)")
201
 
202
  # Measurement units glued to numbers: 100km, 50kg, 25Β°C, 5GB
203
- _RE_UNIT = re.compile(r"(\d+(?:\.\d+)?)\s*(km|kg|mg|ml|gb|mb|kb|tb|hz|khz|mhz|ghz|mph|kph|Β°[cCfF]|[cCfF]Β°|ms|ns|Β΅s)\b",
204
- re.IGNORECASE)
 
 
205
 
206
  # Scale suffixes (uppercase only to avoid ambiguity): 7B, 340M, 1.5K, 2T
207
  # Must NOT be preceded by a letter (so 'MB' is handled by unit regex first)
208
- _RE_SCALE = re.compile(r"(?<![a-zA-Z])(\d+(?:\.\d+)?)\s*([KMBT])(?![a-zA-Z\d])")
209
 
210
  # Scientific notation: 1e-4, 2.5e10, 6.022E23
211
- _RE_SCI = re.compile(r"(?<![a-zA-Z\d])(-?\d+(?:\.\d+)?)[eE]([+-]?\d+)(?![a-zA-Z\d])")
212
 
213
  # Fractions: 1/2, 3/4, 2/3
214
- _RE_FRACTION = re.compile(r"\b(\d+)\s*/\s*(\d+)\b")
215
 
216
  # Decades: 80s, 90s, 1980s, 2020s (number ending in 0 followed by 's')
217
- _RE_DECADE = re.compile(r"\b(\d{1,3})0s\b")
218
 
219
  # Leading decimal (no digit before the dot): .5, .75
220
- _RE_LEAD_DEC = re.compile(r"(?<!\d)\.([\d])")
221
 
222
 
223
  # ─────────────────────────────────────────────
@@ -230,30 +283,32 @@ def expand_abbreviations(text: str) -> str:
230
  .com -> dot com
231
  """
232
  # 1. AI to A.I. (Case sensitive)
233
- text = _RE_AI.sub("A.I.", text)
234
  # 2. .com to dot com
235
- text = _RE_DOT_COM.sub(" dot com", text)
236
  return text
237
 
 
238
  def expand_symbols(text: str) -> str:
239
  """
240
  Translates mathematical and connector symbols to words.
241
  """
242
- text = _RE_PLUS.sub(" plus ", text)
243
- text = _RE_AMPERSAND.sub(" and ", text)
244
- text = _RE_AT_SYMBOL.sub(" at ", text)
245
  return text
246
 
 
247
  def _ordinal_suffix(n: int) -> str:
248
  """Return the ordinal word for n (e.g. 1 β†’ 'first', 5 β†’ 'fifth', 21 β†’ 'twenty-first')."""
249
  word = number_to_words(n)
250
  # For hyphenated compounds like "twenty-one", convert only the last part
251
- if "-" in word:
252
- prefix, last = word.rsplit("-", 1)
253
- joiner = "-"
254
  else:
255
- parts = word.rsplit(" ", 1)
256
- prefix, last, joiner = (parts[0], parts[1], " ") if len(parts) == 2 else ("", parts[0], "")
257
 
258
  # Check exception table
259
  for base, ordinal in _ORDINAL_EXCEPTIONS.items():
@@ -262,14 +317,14 @@ def _ordinal_suffix(n: int) -> str:
262
  break
263
  else:
264
  # General rule
265
- if last.endswith("t"):
266
- last_ord = last + "h"
267
- elif last.endswith("e"):
268
- last_ord = last[:-1] + "th"
269
  else:
270
- last_ord = last + "th"
271
 
272
- return f"{prefix}{joiner}{last_ord}" if prefix else last_ord
273
 
274
 
275
  def expand_ordinals(text: str) -> str:
@@ -283,8 +338,10 @@ def expand_ordinals(text: str) -> str:
283
  "21st century" β†’ "twenty-first century"
284
  "100th day" β†’ "one hundredth day"
285
  """
 
286
  def _replace(m: re.Match) -> str:
287
  return _ordinal_suffix(int(m.group(1)))
 
288
  return _RE_ORDINAL.sub(_replace, text)
289
 
290
 
@@ -297,20 +354,25 @@ def expand_percentages(text: str) -> str:
297
  "3.5% rate" β†’ "three point five percent rate"
298
  "-2% change" β†’ "negative two percent change"
299
  """
 
300
  def _replace(m: re.Match) -> str:
301
- raw = m.group(1).replace(",", "")
302
- if "." in raw:
303
- return float_to_words(float(raw)) + " percent"
304
- return number_to_words(int(raw)) + " percent"
 
305
  return _RE_PERCENT.sub(_replace, text)
306
 
 
307
  def expand_newlines(text: str) -> str:
308
  """Change newlines/returns to a period and space for TTS pausing."""
309
- return _RE_NEWLINE.sub(". ", text)
 
310
 
311
  def expand_tilde(text: str) -> str:
312
  """Change ~ to 'about'."""
313
- return _RE_TILDE.sub("about ", text)
 
314
 
315
  def expand_currency(text: str) -> str:
316
  """
@@ -323,36 +385,38 @@ def expand_currency(text: str) -> str:
323
  "$85K" β†’ "eighty five thousand dollars"
324
  "$2.5M" β†’ "two point five million dollars"
325
  """
 
326
  def _replace(m: re.Match) -> str:
327
  symbol = m.group(1)
328
- raw = m.group(2).replace(",", "")
329
  scale_suffix = m.group(3)
330
- unit = _CURRENCY_SYMBOLS.get(symbol, "")
331
 
332
  # Handle Scaled Currency ($17.5 billion or $17.5B)
333
  if scale_suffix:
334
  # Normalize suffix (e.g., 'B' or 'billion' -> 'billion')
335
  scale_word = _CURRENCY_SCALE_MAP.get(scale_suffix.upper(), scale_suffix.lower())
336
- num = float_to_words(raw) if "." in raw else number_to_words(int(raw))
337
- return f"{num} {scale_word} {unit}{'s' if unit else ''}".strip()
338
 
339
  # Handle Standard Currency ($17.50)
340
- if "." in raw:
341
- int_part, dec_part = raw.split(".", 1)
342
- dec_val = int(dec_part[:2].ljust(2, "0"))
343
  int_words = number_to_words(int(int_part))
344
- result = f"{int_words} {unit}s" if unit else int_words
345
  if dec_val:
346
  cents = number_to_words(dec_val)
347
- result += f" and {cents} cent{'s' if dec_val != 1 else ''}"
348
  else:
349
  val = int(raw)
350
  words = number_to_words(val)
351
- result = f"{words} {unit}{'s' if val != 1 and unit else ''}" if unit else words
352
  return result
353
 
354
  return _RE_CURRENCY.sub(_replace, text)
355
 
 
356
  def expand_time(text: str) -> str:
357
  """
358
  Expand time expressions.
@@ -363,17 +427,19 @@ def expand_time(text: str) -> str:
363
  "9:05 AM" β†’ "nine oh five am"
364
  "12:00pm" β†’ "twelve pm"
365
  """
 
366
  def _replace(m: re.Match) -> str:
367
  h = int(m.group(1))
368
  mins = int(m.group(2))
369
- suffix = (" " + m.group(4).lower()) if m.group(4) else ""
370
  h_words = number_to_words(h)
371
  if mins == 0:
372
- return f"{h_words} hundred{suffix}" if not m.group(4) else f"{h_words}{suffix}"
373
  elif mins < 10:
374
- return f"{h_words} oh {number_to_words(mins)}{suffix}"
375
  else:
376
- return f"{h_words} {number_to_words(mins)}{suffix}"
 
377
  return _RE_TIME.sub(_replace, text)
378
 
379
 
@@ -386,10 +452,12 @@ def expand_ranges(text: str) -> str:
386
  "pages 100-200" β†’ "pages one hundred to two hundred"
387
  "2020-2024" β†’ "twenty twenty to twenty twenty-four"
388
  """
 
389
  def _replace(m: re.Match) -> str:
390
  lo = number_to_words(int(m.group(1)))
391
  hi = number_to_words(int(m.group(2)))
392
- return f"{lo} to {hi}"
 
393
  return _RE_RANGE.sub(_replace, text)
394
 
395
 
@@ -406,7 +474,7 @@ def expand_model_names(text: str) -> str:
406
  "v2.0" stays as "v2.0" (no hyphen β€” handled by number replacement)
407
  "IPv6" stays as "IPv6"
408
  """
409
- return _RE_MODEL_VER.sub(lambda m: f"{m.group(1)} {m.group(2)}", text)
410
 
411
 
412
  def expand_units(text: str) -> str:
@@ -420,21 +488,36 @@ def expand_units(text: str) -> str:
420
  "5GB" β†’ "five gigabytes"
421
  """
422
  _unit_map = {
423
- "km": "kilometers", "kg": "kilograms", "mg": "milligrams",
424
- "ml": "milliliters", "gb": "gigabytes", "mb": "megabytes",
425
- "kb": "kilobytes", "tb": "terabytes",
426
- "hz": "hertz", "khz": "kilohertz", "mhz": "megahertz", "ghz": "gigahertz",
427
- "mph": "miles per hour", "kph": "kilometers per hour",
428
- "ms": "milliseconds", "ns": "nanoseconds", "Β΅s": "microseconds",
429
- "Β°c": "degrees Celsius", "cΒ°": "degrees Celsius",
430
- "Β°f": "degrees Fahrenheit", "fΒ°": "degrees Fahrenheit",
 
 
 
 
 
 
 
 
 
 
 
 
 
431
  }
 
432
  def _replace(m: re.Match) -> str:
433
  raw = m.group(1)
434
  unit = m.group(2).lower()
435
  expanded = _unit_map.get(unit, m.group(2))
436
- num = float_to_words(float(raw)) if "." in raw else number_to_words(int(raw))
437
- return f"{num} {expanded}"
 
438
  return _RE_UNIT.sub(_replace, text)
439
 
440
 
@@ -450,9 +533,9 @@ def expand_roman_numerals(text: str, context_words: bool = True) -> str:
450
  "mix I with V" β†’ left unchanged (ambiguous single letters)
451
  """
452
  _TITLE_WORDS = re.compile(
453
- r"\b(war|chapter|part|volume|act|scene|book|section|article|"
454
- r"king|queen|pope|louis|henry|edward|george|william|james|"
455
- r"phase|round|level|stage|class|type|version|episode|season)\b",
456
  re.IGNORECASE,
457
  )
458
 
@@ -461,10 +544,10 @@ def expand_roman_numerals(text: str, context_words: bool = True) -> str:
461
  if not roman.strip():
462
  return roman
463
  # Skip single ambiguous letters (I, V, X) unless context present
464
- if len(roman) == 1 and roman in "IVX":
465
  # Only expand if preceded by a title word
466
  start = m.start()
467
- preceding = text[max(0, start - 30): start]
468
  if not _TITLE_WORDS.search(preceding):
469
  return roman
470
  try:
@@ -487,8 +570,8 @@ def normalize_leading_decimals(text: str) -> str:
487
  "-.25 adjustment" β†’ "-0.25 adjustment"
488
  """
489
  # Handle -.5 β†’ -0.5 and .5 β†’ 0.5
490
- text = re.sub(r"(?<!\d)(-)\.([\d])", r"\g<1>0.\2", text)
491
- return _RE_LEAD_DEC.sub(r"0.\1", text)
492
 
493
 
494
  def expand_scientific_notation(text: str) -> str:
@@ -500,13 +583,17 @@ def expand_scientific_notation(text: str) -> str:
500
  "2.5e10" β†’ "two point five times ten to the ten"
501
  "6.022E23"β†’ "six point zero two two times ten to the twenty three"
502
  """
 
503
  def _replace(m: re.Match) -> str:
504
  coeff_raw = m.group(1)
505
  exp = int(m.group(2))
506
- coeff_words = float_to_words(coeff_raw) if "." in coeff_raw else number_to_words(int(coeff_raw))
 
 
507
  exp_words = number_to_words(abs(exp))
508
- sign = "negative " if exp < 0 else ""
509
- return f"{coeff_words} times ten to the {sign}{exp_words}"
 
510
  return _RE_SCI.sub(_replace, text)
511
 
512
 
@@ -520,14 +607,14 @@ def expand_scale_suffixes(text: str) -> str:
520
  "1.5K salary" β†’ "one point five thousand salary"
521
  "$100K budget" β†’ "$100K budget" (currency handled upstream)
522
  """
523
- _map = {"K": "thousand", "M": "million", "B": "billion", "T": "trillion"}
524
 
525
  def _replace(m: re.Match) -> str:
526
  raw = m.group(1)
527
  suffix = m.group(2)
528
  scale_word = _map.get(suffix, suffix)
529
- num = float_to_words(raw) if "." in raw else number_to_words(int(raw))
530
- return f"{num} {scale_word}"
531
 
532
  return _RE_SCALE.sub(_replace, text)
533
 
@@ -542,6 +629,7 @@ def expand_fractions(text: str) -> str:
542
  "2/3 done" β†’ "two thirds done"
543
  "5/8 inch" β†’ "five eighths inch"
544
  """
 
545
  def _replace(m: re.Match) -> str:
546
  num = int(m.group(1))
547
  den = int(m.group(2))
@@ -549,14 +637,14 @@ def expand_fractions(text: str) -> str:
549
  return m.group()
550
  num_words = number_to_words(num)
551
  if den == 2:
552
- denom_word = "half" if num == 1 else "halves"
553
  elif den == 4:
554
- denom_word = "quarter" if num == 1 else "quarters"
555
  else:
556
  denom_word = _ordinal_suffix(den)
557
  if num != 1:
558
- denom_word += "s"
559
- return f"{num_words} {denom_word}"
560
 
561
  return _RE_FRACTION.sub(_replace, text)
562
 
@@ -572,18 +660,26 @@ def expand_decades(text: str) -> str:
572
  "'90s music" β†’ "nineties music"
573
  """
574
  _decade_map = {
575
- 0: "hundreds", 1: "tens", 2: "twenties", 3: "thirties", 4: "forties",
576
- 5: "fifties", 6: "sixties", 7: "seventies", 8: "eighties", 9: "nineties",
 
 
 
 
 
 
 
 
577
  }
578
 
579
  def _replace(m: re.Match) -> str:
580
- base = int(m.group(1)) # e.g. 8 for "80s", 198 for "1980s"
581
  decade_digit = base % 10
582
- decade_word = _decade_map.get(decade_digit, "")
583
  if base < 10:
584
  return decade_word
585
- century_part = base // 10 # e.g. 19 for 198
586
- return f"{number_to_words(century_part)} {decade_word}"
587
 
588
  return _RE_DECADE.sub(_replace, text)
589
 
@@ -596,16 +692,26 @@ def expand_ip_addresses(text: str) -> str:
596
  "192.168.1.1" β†’ "one nine two dot one six eight dot one dot one"
597
  "10.0.0.1" β†’ "one zero dot zero dot zero dot one"
598
  """
599
- _d = {"0": "zero", "1": "one", "2": "two", "3": "three", "4": "four",
600
- "5": "five", "6": "six", "7": "seven", "8": "eight", "9": "nine"}
 
 
 
 
 
 
 
 
 
 
601
 
602
  def _octet(s: str) -> str:
603
- return " ".join(_d[c] for c in s)
604
 
605
  def _replace(m: re.Match) -> str:
606
- return " dot ".join(_octet(g) for g in m.groups())
607
 
608
- return re.sub(r"\b(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})\b", _replace, text)
609
 
610
 
611
  def expand_phone_numbers(text: str) -> str:
@@ -617,32 +723,47 @@ def expand_phone_numbers(text: str) -> str:
617
  "555-123-4567" β†’ "five five five one two three four five six seven"
618
  "1-800-555-0199" β†’ "one eight zero zero five five five zero one nine nine"
619
  """
620
- _d = {"0": "zero", "1": "one", "2": "two", "3": "three", "4": "four",
621
- "5": "five", "6": "six", "7": "seven", "8": "eight", "9": "nine"}
 
 
 
 
 
 
 
 
 
 
622
 
623
  def _digits(s: str) -> str:
624
- return " ".join(_d[c] for c in s)
625
 
626
  def _join(*groups) -> str:
627
- return " ".join(_digits(g) for g in groups)
628
 
629
  # Match longest pattern first to avoid partial matches
630
  # 11-digit: 1-800-555-0199
631
- text = re.sub(r"(?<!\d-)(?<!\d)\b(\d{1,2})-(\d{3})-(\d{3})-(\d{4})\b(?!-\d)",
632
- lambda m: _join(*m.groups()), text)
 
 
 
633
  # 10-digit: 555-123-4567
634
- text = re.sub(r"(?<!\d-)(?<!\d)\b(\d{3})-(\d{3})-(\d{4})\b(?!-\d)",
635
- lambda m: _join(*m.groups()), text)
 
636
  # 7-digit local: 555-1234 (not preceded or followed by digit-hyphen to avoid sub-matching)
637
- text = re.sub(r"(?<!\d-)\b(\d{3})-(\d{4})\b(?!-\d)",
638
- lambda m: _join(*m.groups()), text)
639
  return text
640
 
 
641
  def expand_months(text: str) -> str:
642
  """
643
  Expands Jan, Feb, etc. to January, February.
644
  Only triggers if the abbreviation is likely a date.
645
  """
 
646
  def _replace(m: re.Match) -> str:
647
  return _MONTH_MAP.get(m.group(1), m.group(1))
648
 
@@ -650,14 +771,16 @@ def expand_months(text: str) -> str:
650
  text = _RE_MONTHS.sub(_replace, text)
651
 
652
  # 2. May (Special case: only if followed by a digit)
653
- text = _RE_MAY.sub("May", text) # Essentially just ensuring it's treated as a word
654
 
655
  return text
656
 
 
657
  # ─────────────────────────────────────────────
658
  # Core preprocessing functions
659
  # ─────────────────────────────────────────────
660
 
 
661
  def replace_numbers(text: str, replace_floats: bool = True) -> str:
662
  """
663
  Replace all numeric tokens with their word equivalents.
@@ -667,16 +790,18 @@ def replace_numbers(text: str, replace_floats: bool = True) -> str:
667
  "Pi is 3.14" β†’ "Pi is three point one four"
668
  "gpt-3 rocks" β†’ "gpt-3 rocks" (hyphen not treated as minus)
669
  """
 
670
  def _replace(m: re.Match) -> str:
671
- raw = m.group().replace(",", "")
672
  try:
673
- if "." in raw and replace_floats:
674
  # Pass raw string so trailing zeros are preserved ("1.50" β†’ "one point five zero")
675
  return float_to_words(raw)
676
  else:
677
  return number_to_words(int(float(raw)))
678
  except (ValueError, OverflowError):
679
  return m.group()
 
680
  return _RE_NUMBER.sub(_replace, text)
681
 
682
 
@@ -685,50 +810,50 @@ def to_lowercase(text: str) -> str:
685
  return text.lower()
686
 
687
 
688
- def remove_urls(text: str, replacement: str = "") -> str:
689
  """Remove URLs from text."""
690
  return _RE_URL.sub(replacement, text).strip()
691
 
692
 
693
- def remove_emails(text: str, replacement: str = "") -> str:
694
  """Remove email addresses from text."""
695
  return _RE_EMAIL.sub(replacement, text).strip()
696
 
697
 
698
  def remove_html_tags(text: str) -> str:
699
  """Strip HTML tags from text."""
700
- return _RE_HTML.sub(" ", text)
701
 
702
 
703
- def remove_hashtags(text: str, replacement: str = "") -> str:
704
  """Remove hashtags (e.g. #NLP) from text."""
705
  return _RE_HASHTAG.sub(replacement, text)
706
 
707
 
708
- def remove_mentions(text: str, replacement: str = "") -> str:
709
  """Remove @mentions from text."""
710
  return _RE_MENTION.sub(replacement, text)
711
 
712
 
713
  def remove_punctuation(text: str) -> str:
714
  """Remove all punctuation characters."""
715
- return _RE_PUNCT.sub(" ", text)
716
 
717
 
718
  def remove_extra_whitespace(text: str) -> str:
719
  """Collapse multiple whitespace characters into a single space and strip ends."""
720
- return _RE_SPACES.sub(" ", text).strip()
721
 
722
 
723
- def normalize_unicode(text: str, form: str = "NFC") -> str:
724
  """Normalize unicode characters (NFC, NFD, NFKC, or NFKD)."""
725
  return unicodedata.normalize(form, text)
726
 
727
 
728
  def remove_accents(text: str) -> str:
729
  """Remove diacritical marks (accents) from characters."""
730
- nfkd = unicodedata.normalize("NFD", text)
731
- return "".join(c for c in nfkd if unicodedata.category(c) != "Mn")
732
 
733
 
734
  def expand_contractions(text: str) -> str:
@@ -741,18 +866,18 @@ def expand_contractions(text: str) -> str:
741
  "I've" β†’ "I have"
742
  """
743
  contractions = {
744
- r"\bcan't\b": "cannot",
745
- r"\bwon't\b": "will not",
746
- r"\bshan't\b": "shall not",
747
- r"\bain't\b": "is not",
748
- r"\blet's\b": "let us",
749
- r"\b(\w+)n't\b": r"\1 not",
750
- r"\b(\w+)'re\b": r"\1 are",
751
- r"\b(\w+)'ve\b": r"\1 have",
752
- r"\b(\w+)'ll\b": r"\1 will",
753
- r"\b(\w+)'d\b": r"\1 would",
754
- r"\b(\w+)'m\b": r"\1 am",
755
- r"\bit's\b": "it is",
756
  }
757
  for pattern, replacement in contractions.items():
758
  text = re.sub(pattern, replacement, text, flags=re.IGNORECASE)
@@ -768,21 +893,70 @@ def remove_stopwords(text: str, stopwords: set | None = None) -> str:
768
  """
769
  if stopwords is None:
770
  stopwords = {
771
- "a", "an", "the", "and", "or", "but", "in", "on", "at", "to",
772
- "for", "of", "with", "by", "from", "is", "was", "are", "were",
773
- "be", "been", "being", "have", "has", "had", "do", "does", "did",
774
- "will", "would", "could", "should", "may", "might", "this", "that",
775
- "these", "those", "it", "its", "i", "me", "my", "we", "our",
776
- "you", "your", "he", "she", "him", "her", "they", "them", "their",
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
777
  }
778
  tokens = text.split()
779
- return " ".join(t for t in tokens if t.lower() not in stopwords)
780
 
781
 
782
  # ─────────────────────────────────────────────
783
  # Pipeline helper
784
  # ─────────────────────────────────────────────
785
 
 
786
  class TextPreprocessor:
787
  """
788
  Configurable preprocessing pipeline.
@@ -836,7 +1010,7 @@ class TextPreprocessor:
836
  remove_accents: bool = False,
837
  remove_extra_whitespace: bool = True,
838
  ):
839
- self.config = {k: v for k, v in locals().items() if k != "self"}
840
  self._stopwords = stopwords
841
 
842
  def __call__(self, text: str) -> str:
@@ -844,76 +1018,76 @@ class TextPreprocessor:
844
 
845
  def process(self, text: str) -> str:
846
  cfg = self.config
847
- if cfg.get("expand_abbreviations"):
848
  text = expand_abbreviations(text)
849
  text = expand_months(text)
850
- if cfg.get("expand_newlines"):
851
  text = expand_newlines(text)
852
- if cfg.get("expand_symbols"):
853
  text = expand_symbols(text)
854
- if cfg.get("expand_tilde"):
855
  text = expand_tilde(text)
856
- if cfg["normalize_unicode"]:
857
  text = normalize_unicode(text)
858
- if cfg["remove_html"]:
859
  text = remove_html_tags(text)
860
- if cfg["remove_urls"]:
861
  text = remove_urls(text)
862
- if cfg["remove_emails"]:
863
  text = remove_emails(text)
864
- if cfg["remove_hashtags"]:
865
  text = remove_hashtags(text)
866
- if cfg["remove_mentions"]:
867
  text = remove_mentions(text)
868
- if cfg["expand_contractions"]:
869
  text = expand_contractions(text)
870
  # IP addresses before normalize_leading_decimals (IPs contain dots before digits)
871
- if cfg["expand_ip_addresses"]:
872
  text = expand_ip_addresses(text)
873
  # Normalise bare leading decimals early so downstream regexes see "0.5" not ".5"
874
- if cfg["normalize_leading_decimals"]:
875
  text = normalize_leading_decimals(text)
876
  # Expand special forms before generic number replacement
877
- if cfg["expand_currency"]:
878
  text = expand_currency(text)
879
- if cfg["expand_percentages"]:
880
  text = expand_percentages(text)
881
  # Scientific notation before model-name expansion (e.g. "1e-4" contains "e-4")
882
- if cfg["expand_scientific_notation"]:
883
  text = expand_scientific_notation(text)
884
- if cfg["expand_time"]:
885
  text = expand_time(text)
886
- if cfg["expand_ordinals"]:
887
  text = expand_ordinals(text)
888
- if cfg["expand_units"]:
889
  text = expand_units(text)
890
  # Scale suffixes after units (units handles "MB"/"GB"; this handles bare "B"/"M")
891
- if cfg["expand_scale_suffixes"]:
892
  text = expand_scale_suffixes(text)
893
- if cfg["expand_fractions"]:
894
  text = expand_fractions(text)
895
- if cfg["expand_decades"]:
896
  text = expand_decades(text)
897
  # Phone numbers before ranges, otherwise NNN-NNNN is treated as a range
898
- if cfg["expand_phone_numbers"]:
899
  text = expand_phone_numbers(text)
900
- if cfg["expand_ranges"]:
901
  text = expand_ranges(text)
902
- if cfg["expand_model_names"]:
903
  text = expand_model_names(text)
904
- if cfg["expand_roman_numerals"]:
905
  text = expand_roman_numerals(text)
906
- if cfg["replace_numbers"]:
907
- text = replace_numbers(text, replace_floats=cfg["replace_floats"])
908
- if cfg["remove_accents"]:
909
  text = remove_accents(text)
910
- if cfg["remove_punctuation"]:
911
  text = remove_punctuation(text)
912
- if cfg["lowercase"]:
913
  text = to_lowercase(text)
914
- if cfg["remove_stopwords"]:
915
  text = remove_stopwords(text, self._stopwords)
916
- if cfg["remove_extra_whitespace"]:
917
  text = remove_extra_whitespace(text)
918
 
919
  return text
 
11
  # ─────────────────────────────────────────────
12
 
13
  _ONES = [
14
+ '',
15
+ 'one',
16
+ 'two',
17
+ 'three',
18
+ 'four',
19
+ 'five',
20
+ 'six',
21
+ 'seven',
22
+ 'eight',
23
+ 'nine',
24
+ 'ten',
25
+ 'eleven',
26
+ 'twelve',
27
+ 'thirteen',
28
+ 'fourteen',
29
+ 'fifteen',
30
+ 'sixteen',
31
+ 'seventeen',
32
+ 'eighteen',
33
+ 'nineteen',
34
  ]
35
+ _TENS = ['', '', 'twenty', 'thirty', 'forty', 'fifty', 'sixty', 'seventy', 'eighty', 'ninety']
36
+ _SCALE = ['', 'thousand', 'million', 'billion', 'trillion']
37
 
38
  _ORDINAL_EXCEPTIONS = {
39
+ 'one': 'first',
40
+ 'two': 'second',
41
+ 'three': 'third',
42
+ 'four': 'fourth',
43
+ 'five': 'fifth',
44
+ 'six': 'sixth',
45
+ 'seven': 'seventh',
46
+ 'eight': 'eighth',
47
+ 'nine': 'ninth',
48
+ 'twelve': 'twelfth',
49
  }
50
 
51
  _CURRENCY_SYMBOLS = {
52
+ '$': 'dollar',
53
+ '€': 'euro',
54
+ 'Β£': 'pound',
55
+ 'Β₯': 'yen',
56
+ 'β‚Ή': 'rupee',
57
+ 'β‚©': 'won',
58
+ 'β‚Ώ': 'bitcoin',
59
  }
60
 
61
  _CURRENCY_SCALE_MAP = {
62
+ 'K': 'thousand',
63
+ 'M': 'million',
64
+ 'B': 'billion',
65
+ 'T': 'trillion',
66
+ 'thousand': 'thousand',
67
+ 'million': 'million',
68
+ 'billion': 'billion',
69
+ 'trillion': 'trillion',
70
  }
71
 
72
  _ROMAN = [
73
+ (1000, 'M'),
74
+ (900, 'CM'),
75
+ (500, 'D'),
76
+ (400, 'CD'),
77
+ (100, 'C'),
78
+ (90, 'XC'),
79
+ (50, 'L'),
80
+ (40, 'XL'),
81
+ (10, 'X'),
82
+ (9, 'IX'),
83
+ (5, 'V'),
84
+ (4, 'IV'),
85
+ (1, 'I'),
86
  ]
87
+ _RE_ROMAN = re.compile(r'\b(M{0,4})(CM|CD|D?C{0,3})(XC|XL|L?X{0,3})(IX|IV|V?I{0,3})\b')
 
 
88
 
89
 
90
  def _three_digits_to_words(n: int) -> str:
91
  """Convert a number 0–999 to English words."""
92
  if n == 0:
93
+ return ''
94
  parts = []
95
  hundreds = n // 100
96
  remainder = n % 100
97
  if hundreds:
98
+ parts.append(f'{_ONES[hundreds]} hundred')
99
  if remainder < 20:
100
  if remainder:
101
  parts.append(_ONES[remainder])
102
  else:
103
  tens_word = _TENS[remainder // 10]
104
  ones_word = _ONES[remainder % 10]
105
+ parts.append(f'{tens_word}-{ones_word}' if ones_word else tens_word)
106
+ return ' '.join(parts)
107
 
108
 
109
  def number_to_words(n: int) -> str:
 
120
  if not isinstance(n, int):
121
  n = int(n)
122
  if n == 0:
123
+ return 'zero'
124
  if n < 0:
125
+ return f'negative {number_to_words(-n)}'
126
 
127
  # X00–X999 read as "X hundred" (e.g. 1200 β†’ "twelve hundred")
128
  # Exclude exact multiples of 1000 (1000 β†’ "one thousand", not "ten hundred")
129
  if 100 <= n <= 9999 and n % 100 == 0 and n % 1000 != 0:
130
  hundreds = n // 100
131
  if hundreds < 20:
132
+ return f'{_ONES[hundreds]} hundred'
133
 
134
  parts = []
135
  for _i, scale in enumerate(_SCALE):
136
  chunk = n % 1000
137
  if chunk:
138
  chunk_words = _three_digits_to_words(chunk)
139
+ parts.append(f'{chunk_words} {scale}'.strip() if scale else chunk_words)
140
  n //= 1000
141
  if n == 0:
142
  break
143
 
144
+ return ' '.join(reversed(parts))
145
 
146
 
147
+ def float_to_words(value, decimal_sep: str = 'point') -> str:
148
  """
149
  Convert a float (or numeric string) to words, reading decimal digits individually.
150
  Accepts a string to preserve trailing zeros (e.g. "1.50" β†’ "one point five zero").
 
155
  "3.10" β†’ "three point one zero"
156
  1.007 β†’ "one point zero zero seven"
157
  """
158
+ text = value if isinstance(value, str) else f'{value}'
159
+ negative = text.startswith('-')
160
  if negative:
161
  text = text[1:]
162
 
163
+ if '.' in text:
164
+ int_part, dec_part = text.split('.', 1)
165
+ int_words = number_to_words(int(int_part)) if int_part else 'zero'
166
  # Read each decimal digit individually; "0" β†’ "zero"
167
+ digit_map = ['zero'] + _ONES[1:] # index 0 β†’ "zero"
168
+ dec_words = ' '.join(digit_map[int(d)] for d in dec_part)
169
+ result = f'{int_words} {decimal_sep} {dec_words}'
170
  else:
171
  result = number_to_words(int(text))
172
 
173
+ return f'negative {result}' if negative else result
174
 
175
 
176
  def roman_to_int(s: str) -> int:
177
  """Convert a Roman numeral string to an integer."""
178
+ val = {'I': 1, 'V': 5, 'X': 10, 'L': 50, 'C': 100, 'D': 500, 'M': 1000}
 
179
  result = 0
180
  prev = 0
181
  for ch in reversed(s.upper()):
 
189
  # Regex patterns
190
  # ─────────────────────────────────────────────
191
 
192
+ _RE_URL = re.compile(r'https?://\S+|www\.\S+')
193
+ _RE_EMAIL = re.compile(r'\b[\w.+-]+@[\w-]+\.[a-z]{2,}\b', re.IGNORECASE)
194
+ _RE_HASHTAG = re.compile(r'#\w+')
195
+ _RE_MENTION = re.compile(r'@\w+')
196
+ _RE_HTML = re.compile(r'<[^>]+>')
197
+ _RE_PUNCT = re.compile(r'[^\w\s]')
198
+ _RE_SPACES = re.compile(r'\s+')
199
+ _RE_AI = re.compile(r'\bAI\b')
200
+ _RE_DOT_COM = re.compile(r'\.com\b', re.IGNORECASE)
201
+ _RE_PLUS = re.compile(r'\+')
202
+ _RE_AMPERSAND = re.compile(r'&')
203
+ _RE_AT_SYMBOL = re.compile(r'@')
204
+ _RE_NEWLINE = re.compile(r'[\r\n]+')
205
+ _RE_TILDE = re.compile(r'~')
206
 
207
  _MONTH_MAP = {
208
+ 'Jan': 'January',
209
+ 'Feb': 'February',
210
+ 'Mar': 'March',
211
+ 'Apr': 'April',
212
+ 'Jun': 'June',
213
+ 'Jul': 'July',
214
+ 'Aug': 'August',
215
+ 'Sep': 'September',
216
+ 'Sept': 'September',
217
+ 'Oct': 'October',
218
+ 'Nov': 'November',
219
+ 'Dec': 'December',
220
  }
221
 
222
  # Regex looks for Title Case months followed by a period or a digit
223
  # We handle "May" separately because it's a common word.
224
+ _RE_MONTHS = re.compile(r'\b(Jan|Feb|Mar|Apr|Jun|Jul|Aug|Sep|Sept|Oct|Nov|Dec)\.?\b(?=\s*\d|\s*$)')
225
+ _RE_MAY = re.compile(r'\bMay\b(?=\s*\d)') # Only expand May if followed by a number (May 5)
226
 
227
  # Number: do NOT match a leading minus if it is immediately preceded by a letter
228
  # (handles "gpt-3", "gpl-3", "v-2" etc.)
229
+ _RE_NUMBER = re.compile(r'(?<![a-zA-Z])-?[\d,]+(?:\.\d+)?')
230
 
231
  # Ordinals: 1st, 2nd, 3rd, 4th … 21st, 101st …
232
+ _RE_ORDINAL = re.compile(r'\b(\d+)(st|nd|rd|th)\b', re.IGNORECASE)
233
 
234
  # Percentages: 50%, 3.5%
235
+ _RE_PERCENT = re.compile(r'(-?[\d,]+(?:\.\d+)?)\s*%')
236
 
237
  # Currency: $100, €1,200.50, Β£50, $85K, $2.5M (optional scale suffix)
238
  _RE_CURRENCY = re.compile(
239
+ r'([$€£Β₯β‚Ήβ‚©β‚Ώ])\s*([\d,]+(?:\.\d+)?)\s*(million|billion|trillion|thousand|[KMBT])?\b',
240
+ re.IGNORECASE,
241
  )
242
 
243
  # Time: 3:30pm, 14:00, 3:30 AM β€” requires 2-digit minutes so "3:0" (score) doesn't match
244
+ _RE_TIME = re.compile(r'\b(\d{1,2}):(\d{2})(?::(\d{2}))?\s*(am|pm)?\b', re.IGNORECASE)
245
 
246
  # Ranges: 10-20, 100-200 (both sides numeric, hyphen between them)
247
+ _RE_RANGE = re.compile(r'(?<!\w)(\d+)-(\d+)(?!\w)')
248
 
249
  # Version/model names: gpt-3, gpt-3.5, v2.0, Python-3.10, GPL-3
250
  # Letter(s) + hyphen + digit(s) [+ more version parts]
251
+ _RE_MODEL_VER = re.compile(r'\b([a-zA-Z][a-zA-Z0-9]*)-(\d[\d.]*)(?=[^\d.]|$)')
252
 
253
  # Measurement units glued to numbers: 100km, 50kg, 25Β°C, 5GB
254
+ _RE_UNIT = re.compile(
255
+ r'(\d+(?:\.\d+)?)\s*(km|kg|mg|ml|gb|mb|kb|tb|hz|khz|mhz|ghz|mph|kph|Β°[cCfF]|[cCfF]Β°|ms|ns|Β΅s)\b',
256
+ re.IGNORECASE,
257
+ )
258
 
259
  # Scale suffixes (uppercase only to avoid ambiguity): 7B, 340M, 1.5K, 2T
260
  # Must NOT be preceded by a letter (so 'MB' is handled by unit regex first)
261
+ _RE_SCALE = re.compile(r'(?<![a-zA-Z])(\d+(?:\.\d+)?)\s*([KMBT])(?![a-zA-Z\d])')
262
 
263
  # Scientific notation: 1e-4, 2.5e10, 6.022E23
264
+ _RE_SCI = re.compile(r'(?<![a-zA-Z\d])(-?\d+(?:\.\d+)?)[eE]([+-]?\d+)(?![a-zA-Z\d])')
265
 
266
  # Fractions: 1/2, 3/4, 2/3
267
+ _RE_FRACTION = re.compile(r'\b(\d+)\s*/\s*(\d+)\b')
268
 
269
  # Decades: 80s, 90s, 1980s, 2020s (number ending in 0 followed by 's')
270
+ _RE_DECADE = re.compile(r'\b(\d{1,3})0s\b')
271
 
272
  # Leading decimal (no digit before the dot): .5, .75
273
+ _RE_LEAD_DEC = re.compile(r'(?<!\d)\.([\d])')
274
 
275
 
276
  # ─────────────────────────────────────────────
 
283
  .com -> dot com
284
  """
285
  # 1. AI to A.I. (Case sensitive)
286
+ text = _RE_AI.sub('A.I.', text)
287
  # 2. .com to dot com
288
+ text = _RE_DOT_COM.sub(' dot com', text)
289
  return text
290
 
291
+
292
  def expand_symbols(text: str) -> str:
293
  """
294
  Translates mathematical and connector symbols to words.
295
  """
296
+ text = _RE_PLUS.sub(' plus ', text)
297
+ text = _RE_AMPERSAND.sub(' and ', text)
298
+ text = _RE_AT_SYMBOL.sub(' at ', text)
299
  return text
300
 
301
+
302
  def _ordinal_suffix(n: int) -> str:
303
  """Return the ordinal word for n (e.g. 1 β†’ 'first', 5 β†’ 'fifth', 21 β†’ 'twenty-first')."""
304
  word = number_to_words(n)
305
  # For hyphenated compounds like "twenty-one", convert only the last part
306
+ if '-' in word:
307
+ prefix, last = word.rsplit('-', 1)
308
+ joiner = '-'
309
  else:
310
+ parts = word.rsplit(' ', 1)
311
+ prefix, last, joiner = (parts[0], parts[1], ' ') if len(parts) == 2 else ('', parts[0], '')
312
 
313
  # Check exception table
314
  for base, ordinal in _ORDINAL_EXCEPTIONS.items():
 
317
  break
318
  else:
319
  # General rule
320
+ if last.endswith('t'):
321
+ last_ord = last + 'h'
322
+ elif last.endswith('e'):
323
+ last_ord = last[:-1] + 'th'
324
  else:
325
+ last_ord = last + 'th'
326
 
327
+ return f'{prefix}{joiner}{last_ord}' if prefix else last_ord
328
 
329
 
330
  def expand_ordinals(text: str) -> str:
 
338
  "21st century" β†’ "twenty-first century"
339
  "100th day" β†’ "one hundredth day"
340
  """
341
+
342
  def _replace(m: re.Match) -> str:
343
  return _ordinal_suffix(int(m.group(1)))
344
+
345
  return _RE_ORDINAL.sub(_replace, text)
346
 
347
 
 
354
  "3.5% rate" β†’ "three point five percent rate"
355
  "-2% change" β†’ "negative two percent change"
356
  """
357
+
358
  def _replace(m: re.Match) -> str:
359
+ raw = m.group(1).replace(',', '')
360
+ if '.' in raw:
361
+ return float_to_words(float(raw)) + ' percent'
362
+ return number_to_words(int(raw)) + ' percent'
363
+
364
  return _RE_PERCENT.sub(_replace, text)
365
 
366
+
367
  def expand_newlines(text: str) -> str:
368
  """Change newlines/returns to a period and space for TTS pausing."""
369
+ return _RE_NEWLINE.sub('. ', text)
370
+
371
 
372
  def expand_tilde(text: str) -> str:
373
  """Change ~ to 'about'."""
374
+ return _RE_TILDE.sub('about ', text)
375
+
376
 
377
  def expand_currency(text: str) -> str:
378
  """
 
385
  "$85K" β†’ "eighty five thousand dollars"
386
  "$2.5M" β†’ "two point five million dollars"
387
  """
388
+
389
  def _replace(m: re.Match) -> str:
390
  symbol = m.group(1)
391
+ raw = m.group(2).replace(',', '')
392
  scale_suffix = m.group(3)
393
+ unit = _CURRENCY_SYMBOLS.get(symbol, '')
394
 
395
  # Handle Scaled Currency ($17.5 billion or $17.5B)
396
  if scale_suffix:
397
  # Normalize suffix (e.g., 'B' or 'billion' -> 'billion')
398
  scale_word = _CURRENCY_SCALE_MAP.get(scale_suffix.upper(), scale_suffix.lower())
399
+ num = float_to_words(raw) if '.' in raw else number_to_words(int(raw))
400
+ return f'{num} {scale_word} {unit}{"s" if unit else ""}'.strip()
401
 
402
  # Handle Standard Currency ($17.50)
403
+ if '.' in raw:
404
+ int_part, dec_part = raw.split('.', 1)
405
+ dec_val = int(dec_part[:2].ljust(2, '0'))
406
  int_words = number_to_words(int(int_part))
407
+ result = f'{int_words} {unit}s' if unit else int_words
408
  if dec_val:
409
  cents = number_to_words(dec_val)
410
+ result += f' and {cents} cent{"s" if dec_val != 1 else ""}'
411
  else:
412
  val = int(raw)
413
  words = number_to_words(val)
414
+ result = f'{words} {unit}{"s" if val != 1 and unit else ""}' if unit else words
415
  return result
416
 
417
  return _RE_CURRENCY.sub(_replace, text)
418
 
419
+
420
  def expand_time(text: str) -> str:
421
  """
422
  Expand time expressions.
 
427
  "9:05 AM" β†’ "nine oh five am"
428
  "12:00pm" β†’ "twelve pm"
429
  """
430
+
431
  def _replace(m: re.Match) -> str:
432
  h = int(m.group(1))
433
  mins = int(m.group(2))
434
+ suffix = (' ' + m.group(4).lower()) if m.group(4) else ''
435
  h_words = number_to_words(h)
436
  if mins == 0:
437
+ return f'{h_words} hundred{suffix}' if not m.group(4) else f'{h_words}{suffix}'
438
  elif mins < 10:
439
+ return f'{h_words} oh {number_to_words(mins)}{suffix}'
440
  else:
441
+ return f'{h_words} {number_to_words(mins)}{suffix}'
442
+
443
  return _RE_TIME.sub(_replace, text)
444
 
445
 
 
452
  "pages 100-200" β†’ "pages one hundred to two hundred"
453
  "2020-2024" β†’ "twenty twenty to twenty twenty-four"
454
  """
455
+
456
  def _replace(m: re.Match) -> str:
457
  lo = number_to_words(int(m.group(1)))
458
  hi = number_to_words(int(m.group(2)))
459
+ return f'{lo} to {hi}'
460
+
461
  return _RE_RANGE.sub(_replace, text)
462
 
463
 
 
474
  "v2.0" stays as "v2.0" (no hyphen β€” handled by number replacement)
475
  "IPv6" stays as "IPv6"
476
  """
477
+ return _RE_MODEL_VER.sub(lambda m: f'{m.group(1)} {m.group(2)}', text)
478
 
479
 
480
  def expand_units(text: str) -> str:
 
488
  "5GB" β†’ "five gigabytes"
489
  """
490
  _unit_map = {
491
+ 'km': 'kilometers',
492
+ 'kg': 'kilograms',
493
+ 'mg': 'milligrams',
494
+ 'ml': 'milliliters',
495
+ 'gb': 'gigabytes',
496
+ 'mb': 'megabytes',
497
+ 'kb': 'kilobytes',
498
+ 'tb': 'terabytes',
499
+ 'hz': 'hertz',
500
+ 'khz': 'kilohertz',
501
+ 'mhz': 'megahertz',
502
+ 'ghz': 'gigahertz',
503
+ 'mph': 'miles per hour',
504
+ 'kph': 'kilometers per hour',
505
+ 'ms': 'milliseconds',
506
+ 'ns': 'nanoseconds',
507
+ 'Β΅s': 'microseconds',
508
+ 'Β°c': 'degrees Celsius',
509
+ 'cΒ°': 'degrees Celsius',
510
+ 'Β°f': 'degrees Fahrenheit',
511
+ 'fΒ°': 'degrees Fahrenheit',
512
  }
513
+
514
  def _replace(m: re.Match) -> str:
515
  raw = m.group(1)
516
  unit = m.group(2).lower()
517
  expanded = _unit_map.get(unit, m.group(2))
518
+ num = float_to_words(float(raw)) if '.' in raw else number_to_words(int(raw))
519
+ return f'{num} {expanded}'
520
+
521
  return _RE_UNIT.sub(_replace, text)
522
 
523
 
 
533
  "mix I with V" β†’ left unchanged (ambiguous single letters)
534
  """
535
  _TITLE_WORDS = re.compile(
536
+ r'\b(war|chapter|part|volume|act|scene|book|section|article|'
537
+ r'king|queen|pope|louis|henry|edward|george|william|james|'
538
+ r'phase|round|level|stage|class|type|version|episode|season)\b',
539
  re.IGNORECASE,
540
  )
541
 
 
544
  if not roman.strip():
545
  return roman
546
  # Skip single ambiguous letters (I, V, X) unless context present
547
+ if len(roman) == 1 and roman in 'IVX':
548
  # Only expand if preceded by a title word
549
  start = m.start()
550
+ preceding = text[max(0, start - 30) : start]
551
  if not _TITLE_WORDS.search(preceding):
552
  return roman
553
  try:
 
570
  "-.25 adjustment" β†’ "-0.25 adjustment"
571
  """
572
  # Handle -.5 β†’ -0.5 and .5 β†’ 0.5
573
+ text = re.sub(r'(?<!\d)(-)\.([\d])', r'\g<1>0.\2', text)
574
+ return _RE_LEAD_DEC.sub(r'0.\1', text)
575
 
576
 
577
  def expand_scientific_notation(text: str) -> str:
 
583
  "2.5e10" β†’ "two point five times ten to the ten"
584
  "6.022E23"β†’ "six point zero two two times ten to the twenty three"
585
  """
586
+
587
  def _replace(m: re.Match) -> str:
588
  coeff_raw = m.group(1)
589
  exp = int(m.group(2))
590
+ coeff_words = (
591
+ float_to_words(coeff_raw) if '.' in coeff_raw else number_to_words(int(coeff_raw))
592
+ )
593
  exp_words = number_to_words(abs(exp))
594
+ sign = 'negative ' if exp < 0 else ''
595
+ return f'{coeff_words} times ten to the {sign}{exp_words}'
596
+
597
  return _RE_SCI.sub(_replace, text)
598
 
599
 
 
607
  "1.5K salary" β†’ "one point five thousand salary"
608
  "$100K budget" β†’ "$100K budget" (currency handled upstream)
609
  """
610
+ _map = {'K': 'thousand', 'M': 'million', 'B': 'billion', 'T': 'trillion'}
611
 
612
  def _replace(m: re.Match) -> str:
613
  raw = m.group(1)
614
  suffix = m.group(2)
615
  scale_word = _map.get(suffix, suffix)
616
+ num = float_to_words(raw) if '.' in raw else number_to_words(int(raw))
617
+ return f'{num} {scale_word}'
618
 
619
  return _RE_SCALE.sub(_replace, text)
620
 
 
629
  "2/3 done" β†’ "two thirds done"
630
  "5/8 inch" β†’ "five eighths inch"
631
  """
632
+
633
  def _replace(m: re.Match) -> str:
634
  num = int(m.group(1))
635
  den = int(m.group(2))
 
637
  return m.group()
638
  num_words = number_to_words(num)
639
  if den == 2:
640
+ denom_word = 'half' if num == 1 else 'halves'
641
  elif den == 4:
642
+ denom_word = 'quarter' if num == 1 else 'quarters'
643
  else:
644
  denom_word = _ordinal_suffix(den)
645
  if num != 1:
646
+ denom_word += 's'
647
+ return f'{num_words} {denom_word}'
648
 
649
  return _RE_FRACTION.sub(_replace, text)
650
 
 
660
  "'90s music" β†’ "nineties music"
661
  """
662
  _decade_map = {
663
+ 0: 'hundreds',
664
+ 1: 'tens',
665
+ 2: 'twenties',
666
+ 3: 'thirties',
667
+ 4: 'forties',
668
+ 5: 'fifties',
669
+ 6: 'sixties',
670
+ 7: 'seventies',
671
+ 8: 'eighties',
672
+ 9: 'nineties',
673
  }
674
 
675
  def _replace(m: re.Match) -> str:
676
+ base = int(m.group(1)) # e.g. 8 for "80s", 198 for "1980s"
677
  decade_digit = base % 10
678
+ decade_word = _decade_map.get(decade_digit, '')
679
  if base < 10:
680
  return decade_word
681
+ century_part = base // 10 # e.g. 19 for 198
682
+ return f'{number_to_words(century_part)} {decade_word}'
683
 
684
  return _RE_DECADE.sub(_replace, text)
685
 
 
692
  "192.168.1.1" β†’ "one nine two dot one six eight dot one dot one"
693
  "10.0.0.1" β†’ "one zero dot zero dot zero dot one"
694
  """
695
+ _d = {
696
+ '0': 'zero',
697
+ '1': 'one',
698
+ '2': 'two',
699
+ '3': 'three',
700
+ '4': 'four',
701
+ '5': 'five',
702
+ '6': 'six',
703
+ '7': 'seven',
704
+ '8': 'eight',
705
+ '9': 'nine',
706
+ }
707
 
708
  def _octet(s: str) -> str:
709
+ return ' '.join(_d[c] for c in s)
710
 
711
  def _replace(m: re.Match) -> str:
712
+ return ' dot '.join(_octet(g) for g in m.groups())
713
 
714
+ return re.sub(r'\b(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})\b', _replace, text)
715
 
716
 
717
  def expand_phone_numbers(text: str) -> str:
 
723
  "555-123-4567" β†’ "five five five one two three four five six seven"
724
  "1-800-555-0199" β†’ "one eight zero zero five five five zero one nine nine"
725
  """
726
+ _d = {
727
+ '0': 'zero',
728
+ '1': 'one',
729
+ '2': 'two',
730
+ '3': 'three',
731
+ '4': 'four',
732
+ '5': 'five',
733
+ '6': 'six',
734
+ '7': 'seven',
735
+ '8': 'eight',
736
+ '9': 'nine',
737
+ }
738
 
739
  def _digits(s: str) -> str:
740
+ return ' '.join(_d[c] for c in s)
741
 
742
  def _join(*groups) -> str:
743
+ return ' '.join(_digits(g) for g in groups)
744
 
745
  # Match longest pattern first to avoid partial matches
746
  # 11-digit: 1-800-555-0199
747
+ text = re.sub(
748
+ r'(?<!\d-)(?<!\d)\b(\d{1,2})-(\d{3})-(\d{3})-(\d{4})\b(?!-\d)',
749
+ lambda m: _join(*m.groups()),
750
+ text,
751
+ )
752
  # 10-digit: 555-123-4567
753
+ text = re.sub(
754
+ r'(?<!\d-)(?<!\d)\b(\d{3})-(\d{3})-(\d{4})\b(?!-\d)', lambda m: _join(*m.groups()), text
755
+ )
756
  # 7-digit local: 555-1234 (not preceded or followed by digit-hyphen to avoid sub-matching)
757
+ text = re.sub(r'(?<!\d-)\b(\d{3})-(\d{4})\b(?!-\d)', lambda m: _join(*m.groups()), text)
 
758
  return text
759
 
760
+
761
  def expand_months(text: str) -> str:
762
  """
763
  Expands Jan, Feb, etc. to January, February.
764
  Only triggers if the abbreviation is likely a date.
765
  """
766
+
767
  def _replace(m: re.Match) -> str:
768
  return _MONTH_MAP.get(m.group(1), m.group(1))
769
 
 
771
  text = _RE_MONTHS.sub(_replace, text)
772
 
773
  # 2. May (Special case: only if followed by a digit)
774
+ text = _RE_MAY.sub('May', text) # Essentially just ensuring it's treated as a word
775
 
776
  return text
777
 
778
+
779
  # ─────────────────────────────────────────────
780
  # Core preprocessing functions
781
  # ─────────────────────────────────────────────
782
 
783
+
784
  def replace_numbers(text: str, replace_floats: bool = True) -> str:
785
  """
786
  Replace all numeric tokens with their word equivalents.
 
790
  "Pi is 3.14" β†’ "Pi is three point one four"
791
  "gpt-3 rocks" β†’ "gpt-3 rocks" (hyphen not treated as minus)
792
  """
793
+
794
  def _replace(m: re.Match) -> str:
795
+ raw = m.group().replace(',', '')
796
  try:
797
+ if '.' in raw and replace_floats:
798
  # Pass raw string so trailing zeros are preserved ("1.50" β†’ "one point five zero")
799
  return float_to_words(raw)
800
  else:
801
  return number_to_words(int(float(raw)))
802
  except (ValueError, OverflowError):
803
  return m.group()
804
+
805
  return _RE_NUMBER.sub(_replace, text)
806
 
807
 
 
810
  return text.lower()
811
 
812
 
813
+ def remove_urls(text: str, replacement: str = '') -> str:
814
  """Remove URLs from text."""
815
  return _RE_URL.sub(replacement, text).strip()
816
 
817
 
818
+ def remove_emails(text: str, replacement: str = '') -> str:
819
  """Remove email addresses from text."""
820
  return _RE_EMAIL.sub(replacement, text).strip()
821
 
822
 
823
  def remove_html_tags(text: str) -> str:
824
  """Strip HTML tags from text."""
825
+ return _RE_HTML.sub(' ', text)
826
 
827
 
828
+ def remove_hashtags(text: str, replacement: str = '') -> str:
829
  """Remove hashtags (e.g. #NLP) from text."""
830
  return _RE_HASHTAG.sub(replacement, text)
831
 
832
 
833
+ def remove_mentions(text: str, replacement: str = '') -> str:
834
  """Remove @mentions from text."""
835
  return _RE_MENTION.sub(replacement, text)
836
 
837
 
838
  def remove_punctuation(text: str) -> str:
839
  """Remove all punctuation characters."""
840
+ return _RE_PUNCT.sub(' ', text)
841
 
842
 
843
  def remove_extra_whitespace(text: str) -> str:
844
  """Collapse multiple whitespace characters into a single space and strip ends."""
845
+ return _RE_SPACES.sub(' ', text).strip()
846
 
847
 
848
+ def normalize_unicode(text: str, form: str = 'NFC') -> str:
849
  """Normalize unicode characters (NFC, NFD, NFKC, or NFKD)."""
850
  return unicodedata.normalize(form, text)
851
 
852
 
853
  def remove_accents(text: str) -> str:
854
  """Remove diacritical marks (accents) from characters."""
855
+ nfkd = unicodedata.normalize('NFD', text)
856
+ return ''.join(c for c in nfkd if unicodedata.category(c) != 'Mn')
857
 
858
 
859
  def expand_contractions(text: str) -> str:
 
866
  "I've" β†’ "I have"
867
  """
868
  contractions = {
869
+ r"\bcan't\b": 'cannot',
870
+ r"\bwon't\b": 'will not',
871
+ r"\bshan't\b": 'shall not',
872
+ r"\bain't\b": 'is not',
873
+ r"\blet's\b": 'let us',
874
+ r"\b(\w+)n't\b": r'\1 not',
875
+ r"\b(\w+)'re\b": r'\1 are',
876
+ r"\b(\w+)'ve\b": r'\1 have',
877
+ r"\b(\w+)'ll\b": r'\1 will',
878
+ r"\b(\w+)'d\b": r'\1 would',
879
+ r"\b(\w+)'m\b": r'\1 am',
880
+ r"\bit's\b": 'it is',
881
  }
882
  for pattern, replacement in contractions.items():
883
  text = re.sub(pattern, replacement, text, flags=re.IGNORECASE)
 
893
  """
894
  if stopwords is None:
895
  stopwords = {
896
+ 'a',
897
+ 'an',
898
+ 'the',
899
+ 'and',
900
+ 'or',
901
+ 'but',
902
+ 'in',
903
+ 'on',
904
+ 'at',
905
+ 'to',
906
+ 'for',
907
+ 'of',
908
+ 'with',
909
+ 'by',
910
+ 'from',
911
+ 'is',
912
+ 'was',
913
+ 'are',
914
+ 'were',
915
+ 'be',
916
+ 'been',
917
+ 'being',
918
+ 'have',
919
+ 'has',
920
+ 'had',
921
+ 'do',
922
+ 'does',
923
+ 'did',
924
+ 'will',
925
+ 'would',
926
+ 'could',
927
+ 'should',
928
+ 'may',
929
+ 'might',
930
+ 'this',
931
+ 'that',
932
+ 'these',
933
+ 'those',
934
+ 'it',
935
+ 'its',
936
+ 'i',
937
+ 'me',
938
+ 'my',
939
+ 'we',
940
+ 'our',
941
+ 'you',
942
+ 'your',
943
+ 'he',
944
+ 'she',
945
+ 'him',
946
+ 'her',
947
+ 'they',
948
+ 'them',
949
+ 'their',
950
  }
951
  tokens = text.split()
952
+ return ' '.join(t for t in tokens if t.lower() not in stopwords)
953
 
954
 
955
  # ─────────────────────────────────────────────
956
  # Pipeline helper
957
  # ─────────────────────────────────────────────
958
 
959
+
960
  class TextPreprocessor:
961
  """
962
  Configurable preprocessing pipeline.
 
1010
  remove_accents: bool = False,
1011
  remove_extra_whitespace: bool = True,
1012
  ):
1013
+ self.config = {k: v for k, v in locals().items() if k != 'self'}
1014
  self._stopwords = stopwords
1015
 
1016
  def __call__(self, text: str) -> str:
 
1018
 
1019
  def process(self, text: str) -> str:
1020
  cfg = self.config
1021
+ if cfg.get('expand_abbreviations'):
1022
  text = expand_abbreviations(text)
1023
  text = expand_months(text)
1024
+ if cfg.get('expand_newlines'):
1025
  text = expand_newlines(text)
1026
+ if cfg.get('expand_symbols'):
1027
  text = expand_symbols(text)
1028
+ if cfg.get('expand_tilde'):
1029
  text = expand_tilde(text)
1030
+ if cfg['normalize_unicode']:
1031
  text = normalize_unicode(text)
1032
+ if cfg['remove_html']:
1033
  text = remove_html_tags(text)
1034
+ if cfg['remove_urls']:
1035
  text = remove_urls(text)
1036
+ if cfg['remove_emails']:
1037
  text = remove_emails(text)
1038
+ if cfg['remove_hashtags']:
1039
  text = remove_hashtags(text)
1040
+ if cfg['remove_mentions']:
1041
  text = remove_mentions(text)
1042
+ if cfg['expand_contractions']:
1043
  text = expand_contractions(text)
1044
  # IP addresses before normalize_leading_decimals (IPs contain dots before digits)
1045
+ if cfg['expand_ip_addresses']:
1046
  text = expand_ip_addresses(text)
1047
  # Normalise bare leading decimals early so downstream regexes see "0.5" not ".5"
1048
+ if cfg['normalize_leading_decimals']:
1049
  text = normalize_leading_decimals(text)
1050
  # Expand special forms before generic number replacement
1051
+ if cfg['expand_currency']:
1052
  text = expand_currency(text)
1053
+ if cfg['expand_percentages']:
1054
  text = expand_percentages(text)
1055
  # Scientific notation before model-name expansion (e.g. "1e-4" contains "e-4")
1056
+ if cfg['expand_scientific_notation']:
1057
  text = expand_scientific_notation(text)
1058
+ if cfg['expand_time']:
1059
  text = expand_time(text)
1060
+ if cfg['expand_ordinals']:
1061
  text = expand_ordinals(text)
1062
+ if cfg['expand_units']:
1063
  text = expand_units(text)
1064
  # Scale suffixes after units (units handles "MB"/"GB"; this handles bare "B"/"M")
1065
+ if cfg['expand_scale_suffixes']:
1066
  text = expand_scale_suffixes(text)
1067
+ if cfg['expand_fractions']:
1068
  text = expand_fractions(text)
1069
+ if cfg['expand_decades']:
1070
  text = expand_decades(text)
1071
  # Phone numbers before ranges, otherwise NNN-NNNN is treated as a range
1072
+ if cfg['expand_phone_numbers']:
1073
  text = expand_phone_numbers(text)
1074
+ if cfg['expand_ranges']:
1075
  text = expand_ranges(text)
1076
+ if cfg['expand_model_names']:
1077
  text = expand_model_names(text)
1078
+ if cfg['expand_roman_numerals']:
1079
  text = expand_roman_numerals(text)
1080
+ if cfg['replace_numbers']:
1081
+ text = replace_numbers(text, replace_floats=cfg['replace_floats'])
1082
+ if cfg['remove_accents']:
1083
  text = remove_accents(text)
1084
+ if cfg['remove_punctuation']:
1085
  text = remove_punctuation(text)
1086
+ if cfg['lowercase']:
1087
  text = to_lowercase(text)
1088
+ if cfg['remove_stopwords']:
1089
  text = remove_stopwords(text, self._stopwords)
1090
+ if cfg['remove_extra_whitespace']:
1091
  text = remove_extra_whitespace(text)
1092
 
1093
  return text