| """ |
| Shared text processing utilities. |
| """ |
|
|
| from typing import List, Optional |
| import re |
|
|
|
|
| class TextProcessor: |
| """Text processing utilities.""" |
| |
| @staticmethod |
| def split_sentences(text: str) -> List[str]: |
| """Split text into sentences.""" |
| sentences = re.split(r'[.!?]+', text) |
| return [s.strip() for s in sentences if len(s.strip()) > 3] |
| |
| @staticmethod |
| def tokenize_words(text: str) -> List[str]: |
| """Tokenize text into words.""" |
| text_lower = text.lower() |
| |
| for punct in ".,;:!?()[]{}\"\'": |
| text_lower = text_lower.replace(punct, " ") |
| return text_lower.split() |
| |
| @staticmethod |
| def extract_ngrams(tokens: List[str], n: int) -> List[tuple]: |
| """Extract n-grams from token list.""" |
| if len(tokens) < n: |
| return [] |
| return [tuple(tokens[i:i + n]) for i in range(len(tokens) - n + 1)] |
| |
| @staticmethod |
| def filter_valid_texts(texts: List[str]) -> List[str]: |
| """Filter out empty or error texts.""" |
| return [t.strip() for t in texts if t and not t.startswith("ERROR")] |
| |
| @staticmethod |
| def truncate_text(text: str, max_length: int = 500, suffix: str = "...") -> str: |
| """Truncate text to max length.""" |
| if len(text) <= max_length: |
| return text |
| return text[:max_length] + suffix |
