Spaces:
Sleeping
Sleeping
Update utils/DocsLoader.py
Browse files- utils/DocsLoader.py +1 -1
utils/DocsLoader.py
CHANGED
|
@@ -65,7 +65,7 @@ def load_and_chunk(url: str) -> list[Document]:
|
|
| 65 |
if sent.strip():
|
| 66 |
sentence_docs.append(Document(page_content=sent, metadata=doc.metadata))
|
| 67 |
|
| 68 |
-
splitter = RecursiveCharacterTextSplitter(chunk_size=
|
| 69 |
# splitter = SentenceTransformersTokenTextSplitter(model_name = MODEL_DIR,tokens_per_chunk=500, chunk_overlap=80)
|
| 70 |
# return splitter.split_documents(docs)
|
| 71 |
return splitter.split_documents(sentence_docs)
|
|
|
|
| 65 |
if sent.strip():
|
| 66 |
sentence_docs.append(Document(page_content=sent, metadata=doc.metadata))
|
| 67 |
|
| 68 |
+
splitter = RecursiveCharacterTextSplitter(chunk_size=440, chunk_overlap=80)
|
| 69 |
# splitter = SentenceTransformersTokenTextSplitter(model_name = MODEL_DIR,tokens_per_chunk=500, chunk_overlap=80)
|
| 70 |
# return splitter.split_documents(docs)
|
| 71 |
return splitter.split_documents(sentence_docs)
|