Update utils.py

utils.py

@@ -302,6 +302,17 @@ def load_word_with_metadata(file_path):
     
 ################################################    
 #die Inhalte splitten, um in Vektordatenbank entsprechend zu laden als Splits
+# Funktion zum Splitten und Zuweisen der doc_id
+def split_documents_with_id(docs):
+    splits = []
+    for doc in docs:
+        doc_splits = text_splitter.split_text(doc.page_content)
+        for split_content in doc_splits:
+            split_doc = Document(content=split_content, title=doc.metadata["title"], page=doc.metadata["page"], path=doc.metadata["path"], doc_id=doc.doc_id)
+            splits.append(split_doc)
+    return splits
+
+#finally die Splits erzeugen und laden.....
 def document_loading_splitting():
     ##############################
     # Document loading
@@ -335,6 +346,7 @@ def document_loading_splitting():
     # Vorverarbeitung der Dokumente
     preprocessed_docs = []
     original_docs = []
+    
     for doc in docs:
         doc_id = str(uuid.uuid4())  # Erzeuge eine eindeutige ID
         preprocessed_content = preprocess_text(doc.page_content)
@@ -342,7 +354,8 @@ def document_loading_splitting():
         preprocessed_metadata = {
             "title": preprocessed_title,
             "page": doc.metadata["page"],
-            "path": doc.metadata["path"]
+            "path": doc.metadata["path"],
+            "doc_id": doc_id  # Füge die ID in die Metadaten ein
         }
         preprocessed_doc = Document(content=preprocessed_content, title=preprocessed_metadata["title"], page=preprocessed_metadata["page"], path=preprocessed_metadata["path"], doc_id=doc_id)
         original_doc = Document(content=doc.page_content, title=doc.metadata["title"], page=doc.metadata["page"], path=doc.metadata["path"], doc_id=doc_id)
@@ -354,12 +367,13 @@ def document_loading_splitting():
     # Document splitting
     text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)# RecursiveCharacterTextSplitter(chunk_overlap = 150, chunk_size = 1500)
     splits = text_splitter.split_documents(preprocessed_docs)
+
     # Split sowohl für originale als auch für vorverarbeitete Dokumente
-    original_splits = text_splitter.split_documents(original_docs)
-    preprocessed_splits = text_splitter.split_documents(preprocessed_docs)
+    original_splits = split_documents_with_id(original_docs)
+    preprocessed_splits = split_documents_with_id(preprocessed_docs)
     
     # Mapping von vorverarbeiteten Splits zu Originalsplits anhand der IDs
-    split_to_original_mapping = {p_split.id: o_split for p_split, o_split in zip(preprocessed_splits, original_splits)}
+    split_to_original_mapping = {p_split.metadata["doc_id"]: o_split for p_split, o_split in zip(preprocessed_splits, original_splits)}
 
 
     print("Splits...........................")
@@ -720,11 +734,9 @@ class Document:
         self.metadata = {
             "title": title,
             "page": page,
-            "path": path
+            "path": path,
+            "doc_id": doc_id  # Füge die ID in die Metadaten ein
         }
-        self.doc_id = doc_id
-
-
 
 
 def is_stop_word_or_prefix(s: str, stop_words: list) -> bool: