File size: 1,399 Bytes
dff2db9 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 | """
Shared text processing utilities.
"""
from typing import List, Optional
import re
class TextProcessor:
"""Text processing utilities."""
@staticmethod
def split_sentences(text: str) -> List[str]:
"""Split text into sentences."""
sentences = re.split(r'[.!?]+', text)
return [s.strip() for s in sentences if len(s.strip()) > 3]
@staticmethod
def tokenize_words(text: str) -> List[str]:
"""Tokenize text into words."""
text_lower = text.lower()
# Remove punctuation
for punct in ".,;:!?()[]{}\"\'":
text_lower = text_lower.replace(punct, " ")
return text_lower.split()
@staticmethod
def extract_ngrams(tokens: List[str], n: int) -> List[tuple]:
"""Extract n-grams from token list."""
if len(tokens) < n:
return []
return [tuple(tokens[i:i + n]) for i in range(len(tokens) - n + 1)]
@staticmethod
def filter_valid_texts(texts: List[str]) -> List[str]:
"""Filter out empty or error texts."""
return [t.strip() for t in texts if t and not t.startswith("ERROR")]
@staticmethod
def truncate_text(text: str, max_length: int = 500, suffix: str = "...") -> str:
"""Truncate text to max length."""
if len(text) <= max_length:
return text
return text[:max_length] + suffix |