Update utils.py

utils.py

@@ -397,28 +397,40 @@ def document_storage_chroma(splits):
 
 ########################################################
 #Splits für den Vektorstore speichern - bzw. laden
-def save_splits(splits, directory="chroma/kkg", filename="splits.pkl"):
+def save_splits(preprocessed_splits, original_splits, directory="chroma/kkg", preprocessed_filename="preprocessed_splits.pkl", original_filename="original_splits.pkl"):
     # Erstellen des Verzeichnisses, falls es nicht existiert
     if not os.path.exists(directory):
         os.makedirs(directory)
     
-    # Speichern der Splits
-    filepath = os.path.join(directory, filename)
-    with open(filepath, "wb") as f:
-        pickle.dump(splits, f)
+    # Speichern der vorverarbeiteten Splits
+    preprocessed_filepath = os.path.join(directory, preprocessed_filename)
+    with open(preprocessed_filepath, "wb") as f:
+        pickle.dump(preprocessed_splits, f)
     
-    # Hochladen der Splits-Datei zum Hugging Face Space
-    upload_file_to_huggingface(filepath, f"{directory}/{filename}")
+    # Speichern der Originalsplits
+    original_filepath = os.path.join(directory, original_filename)
+    with open(original_filepath, "wb") as f:
+        pickle.dump(original_splits, f)
+    
+    # Hochladen der Splits-Dateien zum Hugging Face Space
+    upload_file_to_huggingface(preprocessed_filepath, f"{directory}/{preprocessed_filename}")
+    upload_file_to_huggingface(original_filepath, f"{directory}/{original_filename}")
 
-def load_splits(directory="chroma/kkg", filename="splits.pkl"):
+def load_splits(directory="chroma/kkg", preprocessed_filename="preprocessed_splits.pkl", original_filename="original_splits.pkl"):
     # Vollständigen Pfad zur Datei erstellen
-    filepath = os.path.join(directory, filename)
+    preprocessed_filepath = os.path.join(directory, preprocessed_filename)
+    original_filepath = os.path.join(directory, original_filename)
     
-    # Laden der Splits aus der Datei
-    if os.path.exists(filepath):
-        with open(filepath, "rb") as f:
-            return pickle.load(f)
-    return None
+    # Laden der vorverarbeiteten Splits aus der Datei
+    if os.path.exists(preprocessed_filepath) and os.path.exists(original_filepath):
+        with open(preprocessed_filepath, "rb") as f:
+            preprocessed_splits = pickle.load(f)
+        
+        with open(original_filepath, "rb") as f:
+            original_splits = pickle.load(f)
+        
+        return preprocessed_splits, original_splits
+    return None, None
 
 ########################################################
 #Vektorstore speichern - bzw. laden