backend/utils/splitter.py

class CharacterTextSplitter:
    def __init__(
        self,
        chunk_size: int = 1000,
        chunk_overlap: int = 200,
    ):
        assert chunk_size > chunk_overlap, (
            "Chunk size must be greater than chunk overlap"
        )

        self.chunk_size = chunk_size
        self.chunk_overlap = chunk_overlap

    def split(self, text: str) -> list[str]:
        chunks = []
        for i in range(0, len(text), self.chunk_size - self.chunk_overlap):
            chunks.append(text[i : i + self.chunk_size])
        return chunks

    def split_texts(self, texts: list[str]) -> list[str]:
        chunks = []
        for text in texts:
            chunks.extend(self.split(text))
        return chunks