Update utils.py

utils.py

@@ -11,11 +11,12 @@ def split_with_source(text, source):
     splitter = CharacterTextSplitter(
         separator = "\n",
         chunk_size = 256,
-        chunk_overlap  = 72,
+        chunk_overlap  = 0,
         length_function = len,
         add_start_index = True,
     )
     documents = splitter.create_documents([text])
+    print(documents)
     for doc in documents:
         doc.metadata["source"] = source
         # print(doc.metadata)
@@ -44,6 +45,7 @@ def get_document_from_raw_text():
     for i in files:
         file_path = i
         with open(os.path.join(os.path.join(os.getcwd(), "raw_data"),file_path), 'r', encoding="utf-8") as file:
+            # Xử lý bằng text_spliter
             # Tiền xử lý văn bản
             content = file.read().replace('\n\n', "\n")
             # content = ''.join(content.split('.'))
@@ -51,22 +53,28 @@ def get_document_from_raw_text():
             texts = split_with_source(new_doc, i)
             documents = documents + texts
 
+            ##Xử lý mỗi khi xuống dòng
+            # for line in file:
+            #     # Loại bỏ khoảng trắng thừa và ký tự xuống dòng ở đầu và cuối mỗi dòng
+            #     line = line.strip()
+            #     documents.append(Document(page_content=line, metadata={"source": i}))
+    print(documents)
     return documents
 
 def load_the_embedding_retrieve(is_ready = False, k = 3, model= 'sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2'):
+    embeddings = HuggingFaceEmbeddings(model_name=model)
     if is_ready:
-        embeddings = HuggingFaceEmbeddings(model_name=model)
         retriever = Chroma(persist_directory=os.path.join(os.getcwd(), "Data"), embedding_function=embeddings).as_retriever(
             search_kwargs={"k": k}
         )
     else:
-
         documents = get_document_from_raw_text()
-
-        retriever = Chroma.from_documents(documents, embedding=model).as_retriever(
+        print(type(documents))
+        retriever = Chroma.from_documents(documents, embeddings).as_retriever(
             search_kwargs={"k": k}
         )
 
+
     return retriever
 
 def load_the_bm25_retrieve(k = 3):