Update utils.py

utils.py

@@ -329,10 +329,24 @@ def document_loading_splitting():
     # Load YouTube
     #loader = GenericLoader(YoutubeAudioLoader([YOUTUBE_URL_1,YOUTUBE_URL_2], PATH_WORK + YOUTUBE_DIR), OpenAIWhisperParser())
     #docs.extend(loader.load())
+
+
+    # Vorverarbeitung der Dokumente - passend zu dem der Prompts...
+    preprocessed_docs = []
+    for doc in docs:
+        preprocessed_content = preprocess_text(doc.page_content)
+        preprocessed_title = preprocess_text(doc.metadata["title"])
+        preprocessed_metadata = {
+            "title": preprocessed_title,
+            "page": doc.metadata["page"],
+            "path": doc.metadata["path"]
+        }
+        preprocessed_docs.append(Document(metadata=preprocessed_metadata, page_content=preprocessed_content))
+    
     ################################
     # Document splitting
     text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)# RecursiveCharacterTextSplitter(chunk_overlap = 150, chunk_size = 1500)
-    splits = text_splitter.split_documents(docs)
+    splits = text_splitter.split_documents(preprocessed_docs)
     print("Splits...........................")
     for split in splits:
         if 'DIVIS' in split.page_content: