Update utils.py

utils.py

@@ -186,8 +186,6 @@ modell_rag = DistilBertForQuestionAnswering.from_pretrained(HF_MODELL)
 tokenizer_rag = DistilBertTokenizer.from_pretrained(HF_MODELL)
 qa_pipeline = pipeline("question-answering", model=modell_rag, tokenizer=tokenizer_rag)
 
-
-
 HF_MODELL ="EleutherAI/gpt-neo-2.7B"
 modell_rag = GPTNeoForCausalLM.from_pretrained(HF_MODELL)
 tokenizer_rag = GPT2Tokenizer.from_pretrained(HF_MODELL)
@@ -280,7 +278,8 @@ def access_pdf(self, filename):
         )
 
         return temp_path
-
+    
+ """
 #besseren directory Loader als CustomLoader definieren, der den inhalt des dokuemnts, die seitenzahlen, die überschriften und die pfadezu den dokumenten extrahieren
 def  create_custom_loader(file_type, file_list): #create_directory_loader(file_type, directory_path):
     loaders = {
@@ -290,7 +289,7 @@ def  create_custom_loader(file_type, file_list): #create_directory_loader(file_t
 
     
     class CustomLoader:
-        """
+       
         def __init__(self, directory_path, file_type, loader_func):
             self.directory_path = directory_path
             self.file_type = file_type
@@ -306,35 +305,14 @@ def  create_custom_loader(file_type, file_list): #create_directory_loader(file_t
             return documents
 
     return CustomLoader(directory_path, file_type, loaders[file_type])
-        """
-
-    def __init__(self, file_type, file_list, loader_func):
-        self.file_type = file_type
-        self.file_list = file_list
-        self.loader_func = loader_func
-
-    def load(self):
-        documents = []
-        for file_path in self.file_list:
-            with tempfile.NamedTemporaryFile(delete=False, suffix=self.file_type) as temp_file:
-                temp_path = temp_file.name
-            
-            # Datei aus dem Hugging Face Space herunterladen
-            hf_hub_download(
-                repo_id=STORAGE_REPO_ID,
-                filename=file_path,
-                repo_type="space",
-                local_dir=os.path.dirname(temp_path),
-                local_dir_use_symlinks=False,
-                token=hf_token
-            )
-
-            documents.extend(self.loader_func(temp_path))
-            
-            # Temporäre Datei löschen
-            os.unlink(temp_path)
-        return documents
+"""
 
+    
+def create_custom_loader(file_type, file_list):
+    loaders = {
+        '.pdf': load_pdf_with_metadata,
+        '.docx': load_word_with_metadata,
+    }
     return CustomLoader(file_type, file_list, loaders[file_type])
 
 ################################################
@@ -906,6 +884,15 @@ def get_filename(file_pfad):
     return result
 
 
+def is_stop_word_or_prefix(s: str, stop_words: list) -> bool:
+    for stop_word in stop_words:
+        if s.endswith(stop_word):
+            return True
+        for i in range(1, len(stop_word)):
+            if s.endswith(stop_word[:i]):
+                return True
+    return False
+
 #################################################
 #Klasse mit zuständen - z.B. für interrupt wenn Stop gedrückt...
 #################################################
@@ -932,14 +919,35 @@ class Document:
         }
 
 
-def is_stop_word_or_prefix(s: str, stop_words: list) -> bool:
-    for stop_word in stop_words:
-        if s.endswith(stop_word):
-            return True
-        for i in range(1, len(stop_word)):
-            if s.endswith(stop_word[:i]):
-                return True
-    return False
+##########################################
+#Class für die Directory Loader - um sie anzupassen
+##########################################
+class CustomLoader:
+    def __init__(self, file_type, file_list, loader_func):
+        self.file_type = file_type
+        self.file_list = file_list
+        self.loader_func = loader_func
+
+    def load(self):
+        documents = []
+        for file_path in self.file_list:
+            with tempfile.NamedTemporaryFile(delete=False, suffix=self.file_type) as temp_file:
+                temp_path = temp_file.name
+            
+            # Datei aus dem Hugging Face Space herunterladen
+            hf_hub_download(
+                repo_id=STORAGE_REPO_ID,
+                filename=file_path,
+                repo_type="space",
+                local_dir=os.path.dirname(temp_path),
+                local_dir_use_symlinks=False,
+                token=hf_token
+            )
+            documents.extend(self.loader_func(temp_path))
+            os.unlink(temp_path)
+        return documents
+
+