from langchain.document_loaders import TextLoader
from langchain.text_splitter import RecursiveCharacterTextSplitter
import os


def load_documents(directory):
    docs = []
    for filename in os.listdir(directory):
        if filename.endswith(".txt") or filename.endswith(".md"):
            loader = TextLoader(os.path.join(
                directory, filename), encoding='utf-8')
            docs.extend(loader.load())
    return docs


def split_documents(documents, chunk_size=1000, chunk_overlap=200, separators=["\n\n", "\n", " ", ""], length_function=len):
    splitter = RecursiveCharacterTextSplitter(
        chunk_size=chunk_size, chunk_overlap=chunk_overlap, separators=separators, length_function=length_function)
    return splitter.split_documents(documents)