text_splitter.py

from langchain_text_splitters import RecursiveCharacterTextSplitter

class TextSplitter:
    def __init__(self, chunk_size=1024, chunk_overlap=100):
        """
        Initialize the TextSplitter with a specific chunk size and overlap.

        Args:
            chunk_size (int): The size of each text chunk.
            chunk_overlap (int): The overlap size between chunks.
        """
        self.text_splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)

    def split_documents(self, documents):
        """
        Split the provided documents into chunks based on the chunk size and overlap.

        Args:
            documents (list): A list of documents to be split.

        Returns:
            A list of split documents.

        Exceptions:
            Prints an error message if splitting documents fails.
        """
        try:
            return self.text_splitter.split_documents(documents)
        except Exception as e:
            print(f"Error splitting documents: {e}")