|
|
|
|
|
|
|
|
from langchain_text_splitters import RecursiveCharacterTextSplitter |
|
|
|
|
|
CHUNK_SIZE = 1500 |
|
|
CHUNK_OVERLAP = 200 |
|
|
|
|
|
def split_documents(docs): |
|
|
splitter = RecursiveCharacterTextSplitter( |
|
|
chunk_size=CHUNK_SIZE, |
|
|
chunk_overlap=CHUNK_OVERLAP, |
|
|
separators=["\n\n", "\n", ". ", " ", ""], |
|
|
) |
|
|
chunks = splitter.split_documents(docs) |
|
|
|
|
|
for c in chunks: |
|
|
c.metadata["chunk_size"] = CHUNK_SIZE |
|
|
c.metadata["chunk_overlap"] = CHUNK_OVERLAP |
|
|
|
|
|
return chunks |
|
|
|
|
|
if __name__ == "__main__": |
|
|
from load_documents import load_documents |
|
|
docs = load_documents() |
|
|
chunks = split_documents(docs) |
|
|
print("Docs:", len(docs), "Chunks:", len(chunks)) |
|
|
print(chunks[0].page_content[:300], chunks[0].metadata) |
|
|
|
