Update utils/DocsLoader.py

utils/DocsLoader.py

@@ -57,15 +57,17 @@ def load_and_chunk(url: str) -> list[Document]:
 
     else:
         raise HTTPException(400, f"Unsupported document type: {content_type}")
-    # --- Step 1: Sentence split ---
-    sentence_docs = []
-    for doc in docs:
-        sentences = nltk.sent_tokenize(doc.page_content)
-        for sent in sentences:
-            if sent.strip():
-                sentence_docs.append(Document(page_content=sent, metadata=doc.metadata))
+    # # --- Step 1: Sentence split ---
+    # sentence_docs = []
+    # for doc in docs:
+    #     sentences = nltk.sent_tokenize(doc.page_content)
+    #     for sent in sentences:
+    #         if sent.strip():
+    #             sentence_docs.append(Document(page_content=sent, metadata=doc.metadata))
+    full_text = "\n".join([doc.page_content for doc in docs])  
 
-    splitter = RecursiveCharacterTextSplitter(chunk_size=440, chunk_overlap=80)
+    splitter = RecursiveCharacterTextSplitter(chunk_size=800, chunk_overlap=150)
     # splitter = SentenceTransformersTokenTextSplitter(model_name = MODEL_DIR,tokens_per_chunk=500, chunk_overlap=80)
     # return splitter.split_documents(docs)
-    return splitter.split_documents(sentence_docs)
+    # return splitter.split_documents([Document(page_content=full_text)])
+    return splitter.create_documents([full_text])