backend/app/services/normalize.py

import re
from hashlib import sha256
from typing import Optional


_MIN_DOC_CHARS = 200


def normalize_text(text: str) -> str:
    """Normalize text by collapsing whitespace and stripping edges."""
    # Replace multiple whitespace (including newlines) with a single space
    normalized = re.sub(r"\s+", " ", text or "").strip()
    return normalized


def is_valid_document(text: str, min_chars: int = _MIN_DOC_CHARS) -> bool:
    """Return True if the text is long enough to be considered useful."""
    return len(text) >= min_chars


def make_doc_id(source: str, title: str, url: Optional[str] = None) -> str:
    """Create a stable SHA256-based document id from source, title, and URL."""
    base = f"{source}|{title}|{url or ''}"
    digest = sha256(base.encode("utf-8")).hexdigest()
    return digest