app/services/document_loader.py

import logging
from pathlib import Path
from typing import List

from pypdf import PdfReader

logger = logging.getLogger(__name__)

def _clean_text(text: str) -> str:
    lines = [line.strip() for line in text.splitlines() if line.strip()]
    merged = " ".join(lines)
    return " ".join(merged.split())


def load_documents(docs_dir: Path) -> List[dict]:
    docs: List[dict] = []
    if not docs_dir.exists():
        return docs

    for file_path in sorted(docs_dir.rglob("*")):
        if not file_path.is_file():
            continue

        try:
            suffix = file_path.suffix.lower()
            if suffix == ".txt":
                raw = file_path.read_text(encoding="utf-8", errors="ignore")
                cleaned = _clean_text(raw)
                if cleaned:
                    docs.append({"source": str(file_path), "text": cleaned})
            elif suffix == ".pdf":
                reader = PdfReader(str(file_path))
                pages = []
                for page in reader.pages:
                    pages.append(page.extract_text() or "")
                cleaned = _clean_text("\n".join(pages))
                if cleaned:
                    docs.append({"source": str(file_path), "text": cleaned})
        except Exception as e:
            logger.error(f"Error loading document {file_path}: {e}")
            continue

    return docs