Update utils.py

utils.py

@@ -306,9 +306,10 @@ def load_word_with_metadata(file_path):
 def split_documents_with_id(docs, text_splitter):
     splits = []
     for doc in docs:
-        doc_splits = text_splitter.split_text(doc.page_content)
+        doc_splits = text_splitter.split_text(f"{doc.metadata['title']} {doc.page_content}")
         for split_content in doc_splits:
-            split_doc = Document(content=split_content, title=doc.metadata["title"], page=doc.metadata["page"], path=doc.metadata["path"], doc_id=doc.metadata["doc_id"])
+            split_id = str(uuid.uuid4())  # Erzeuge eine eindeutige ID für jeden Split
+            split_doc = Document(content=split_content, title=doc.metadata["title"], page=doc.metadata["page"], path=doc.metadata["path"], split_id=split_id)
             splits.append(split_doc)
     return splits
 
@@ -342,38 +343,26 @@ def document_loading_splitting():
     #loader = GenericLoader(YoutubeAudioLoader([YOUTUBE_URL_1,YOUTUBE_URL_2], PATH_WORK + YOUTUBE_DIR), OpenAIWhisperParser())
     #docs.extend(loader.load())
 
-
-    # Vorverarbeitung der Dokumente
-    preprocessed_docs = []
-    original_docs = []
-    
-    for doc in docs:
-        doc_id = str(uuid.uuid4())  # Erzeuge eine eindeutige ID
-        preprocessed_content = preprocess_text(doc.page_content)
-        preprocessed_title = preprocess_text(doc.metadata["title"])
-        preprocessed_metadata = {
-            "title": preprocessed_title,
-            "page": doc.metadata["page"],
-            "path": doc.metadata["path"],
-            "doc_id": doc_id  # Füge die ID in die Metadaten ein
-        }
-        preprocessed_doc = Document(content=preprocessed_content, title=preprocessed_metadata["title"], page=preprocessed_metadata["page"], path=preprocessed_metadata["path"], doc_id=doc_id)
-        original_doc = Document(content=doc.page_content, title=doc.metadata["title"], page=doc.metadata["page"], path=doc.metadata["path"], doc_id=doc_id)
-        preprocessed_docs.append(preprocessed_doc)
-        original_docs.append(original_doc)
-        print("orgin doc....................................."+str(original_doc))
     
     ################################
     # Document splitting
     text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)# RecursiveCharacterTextSplitter(chunk_overlap = 150, chunk_size = 1500)
     #splits = text_splitter.split_documents(preprocessed_docs)
 
-    # Split sowohl für originale als auch für vorverarbeitete Dokumente
-    original_splits = split_documents_with_id(original_docs, text_splitter)
-    preprocessed_splits = split_documents_with_id(preprocessed_docs, text_splitter)
+     # Vorverarbeitung der Dokumente
+    # Split der Originaldokumente
+    original_splits = split_documents_with_id(docs, text_splitter)
+
+    # Vorverarbeitung der Originalsplits
+    preprocessed_splits = []
+    for split in original_splits:
+        preprocessed_content = preprocess_text(split.page_content)
+        preprocessed_split = Document(content=preprocessed_content, title=split.metadata["title"], page=split.metadata["page"], path=split.metadata["path"], split_id=split.metadata["split_id"])
+        preprocessed_splits.append(preprocessed_split)
+
     
-    # Mapping von vorverarbeiteten Splits zu Originalsplits anhand der IDs
-    split_to_original_mapping = {p_split.metadata["doc_id"]: o_split for p_split, o_split in zip(preprocessed_splits, original_splits)}
+    # Mapping von vorverarbeiteten Splits zu Originalsplits anhand der split_ids
+    split_to_original_mapping = {p_split.metadata["split_id"]: o_split for p_split, o_split in zip(preprocessed_splits, original_splits)}
 
 
     print("Splits...........................")
@@ -381,7 +370,7 @@ def document_loading_splitting():
         if 'divis' in split.page_content:
             print("DIVIS found in chunk:", split)
             
-    return preprocessed_splits, split_to_original_mapping
+    return preprocessed_splits, original_splits, split_to_original_mapping
 
 ###########################################
 #Chroma DB die splits ablegen - vektorisiert...
@@ -580,8 +569,6 @@ def extract_document_info(documents):
         else:
             download_link = doc_path
             
-        # Prüfe, ob doc_id existiert und weise einen Standardwert zu, falls nicht
-        id = getattr(doc, 'doc_id', None)
         
         info = {
             'content': doc.page_content,
@@ -590,7 +577,6 @@ def extract_document_info(documents):
             'seite': doc.metadata.get("page", "Unbekannte Seite"),
             'pfad': doc_path,
             'download_link': download_link,
-            'id': id
         }
         extracted_info.append(info)
     return extracted_info
@@ -735,7 +721,7 @@ class Document:
             "title": title,
             "page": page,
             "path": path,
-            "doc_id": doc_id  # Füge die ID in die Metadaten ein
+            "split_id": split_id  # Füge die ID in die Metadaten ein
         }