Spaces:
Runtime error
Runtime error
Commit ·
dfc2f0a
1
Parent(s): 1757d87
update chunk_size
Browse files
app.py
CHANGED
|
@@ -355,7 +355,7 @@ def process_documents_3(ignored_files: List[str] = []) -> List[Document]:
|
|
| 355 |
print("No new documents to load")
|
| 356 |
exit(0)
|
| 357 |
print(f"Loaded {len(documents)} new documents from {source_directory}")
|
| 358 |
-
text_splitter = RecursiveCharacterTextSplitter(chunk_size=
|
| 359 |
texts = text_splitter.split_documents(documents)
|
| 360 |
print(f"Split into {len(texts)} chunks of text (max. {chunk_size} tokens each)")
|
| 361 |
return texts
|
|
|
|
| 355 |
print("No new documents to load")
|
| 356 |
exit(0)
|
| 357 |
print(f"Loaded {len(documents)} new documents from {source_directory}")
|
| 358 |
+
text_splitter = RecursiveCharacterTextSplitter(chunk_size=8000, chunk_overlap=1000)
|
| 359 |
texts = text_splitter.split_documents(documents)
|
| 360 |
print(f"Split into {len(texts)} chunks of text (max. {chunk_size} tokens each)")
|
| 361 |
return texts
|