app/services/chunker.py

from typing import List, Dict


def _tokenize(text: str) -> List[str]:
    return text.split()


def chunk_documents(
    documents: List[Dict[str, str]],
    chunk_size_tokens: int = 420,
    chunk_overlap_tokens: int = 80,
) -> List[Dict[str, str]]:
    chunks: List[Dict[str, str]] = []
    step = max(1, chunk_size_tokens - chunk_overlap_tokens)

    for doc in documents:
        tokens = _tokenize(doc["text"])
        if not tokens:
            continue

        i = 0
        chunk_id = 0
        while i < len(tokens):
            window = tokens[i : i + chunk_size_tokens]
            if not window:
                break

            chunk_text = " ".join(window)
            chunks.append(
                {
                    "id": f"{doc['source']}::chunk_{chunk_id}",
                    "source": doc["source"],
                    "text": chunk_text,
                }
            )
            chunk_id += 1
            i += step

    return chunks