app/utils/text_utils.py

"""Text utility functions for sanitization and normalization."""

import re


def sanitize_text(text: str) -> str:
    """Strip null bytes and normalize whitespace."""
    text = text.replace("\x00", "")
    text = re.sub(r"[ \t]+", " ", text)
    text = re.sub(r"\n{3,}", "\n\n", text)
    return text.strip()


def estimate_reading_level(text: str) -> str:
    """Rough estimate of text complexity based on word length."""
    words = text.split()
    if not words:
        return "unknown"
    avg_len = sum(len(w) for w in words) / len(words)
    if avg_len < 4:
        return "simple"
    elif avg_len < 6:
        return "moderate"
    else:
        return "complex"