"""
Shared text processing utilities.
"""

from typing import List, Optional
import re


class TextProcessor:
    """Text processing utilities."""
    
    @staticmethod
    def split_sentences(text: str) -> List[str]:
        """Split text into sentences."""
        sentences = re.split(r'[.!?]+', text)
        return [s.strip() for s in sentences if len(s.strip()) > 3]
    
    @staticmethod
    def tokenize_words(text: str) -> List[str]:
        """Tokenize text into words."""
        text_lower = text.lower()
        # Remove punctuation
        for punct in ".,;:!?()[]{}\"\'":
            text_lower = text_lower.replace(punct, " ")
        return text_lower.split()
    
    @staticmethod
    def extract_ngrams(tokens: List[str], n: int) -> List[tuple]:
        """Extract n-grams from token list."""
        if len(tokens) < n:
            return []
        return [tuple(tokens[i:i + n]) for i in range(len(tokens) - n + 1)]
    
    @staticmethod
    def filter_valid_texts(texts: List[str]) -> List[str]:
        """Filter out empty or error texts."""
        return [t.strip() for t in texts if t and not t.startswith("ERROR")]
    
    @staticmethod
    def truncate_text(text: str, max_length: int = 500, suffix: str = "...") -> str:
        """Truncate text to max length."""
        if len(text) <= max_length:
            return text
        return text[:max_length] + suffix