Spaces:
Sleeping
Sleeping
| from langchain.document_loaders import TextLoader | |
| from langchain.text_splitter import RecursiveCharacterTextSplitter | |
| import os | |
| def load_documents(directory): | |
| docs = [] | |
| for filename in os.listdir(directory): | |
| if filename.endswith(".txt") or filename.endswith(".md"): | |
| loader = TextLoader(os.path.join( | |
| directory, filename), encoding='utf-8') | |
| docs.extend(loader.load()) | |
| return docs | |
| def split_documents(documents, chunk_size=1000, chunk_overlap=200, separators=["\n\n", "\n", " ", ""], length_function=len): | |
| splitter = RecursiveCharacterTextSplitter( | |
| chunk_size=chunk_size, chunk_overlap=chunk_overlap, separators=separators, length_function=length_function) | |
| return splitter.split_documents(documents) | |