singhankur01 commited on
Commit
44e5786
·
verified ·
1 Parent(s): d3e01cd

Update utils/DocsLoader.py

Browse files
Files changed (1) hide show
  1. utils/DocsLoader.py +1 -1
utils/DocsLoader.py CHANGED
@@ -65,7 +65,7 @@ def load_and_chunk(url: str) -> list[Document]:
65
  if sent.strip():
66
  sentence_docs.append(Document(page_content=sent, metadata=doc.metadata))
67
 
68
- splitter = RecursiveCharacterTextSplitter(chunk_size=600, chunk_overlap=80)
69
  # splitter = SentenceTransformersTokenTextSplitter(model_name = MODEL_DIR,tokens_per_chunk=500, chunk_overlap=80)
70
  # return splitter.split_documents(docs)
71
  return splitter.split_documents(sentence_docs)
 
65
  if sent.strip():
66
  sentence_docs.append(Document(page_content=sent, metadata=doc.metadata))
67
 
68
+ splitter = RecursiveCharacterTextSplitter(chunk_size=440, chunk_overlap=80)
69
  # splitter = SentenceTransformersTokenTextSplitter(model_name = MODEL_DIR,tokens_per_chunk=500, chunk_overlap=80)
70
  # return splitter.split_documents(docs)
71
  return splitter.split_documents(sentence_docs)