Update utils.py
Browse files
utils.py
CHANGED
|
@@ -396,8 +396,7 @@ def document_storage_chroma(splits):
|
|
| 396 |
return vectorstore
|
| 397 |
|
| 398 |
########################################################
|
| 399 |
-
#Vektorstore speichern - bzw. laden
|
| 400 |
-
"""
|
| 401 |
def save_splits_and_metadata(splits, directory="chroma/kkg", filename="splits_and_metadata.pkl"):
|
| 402 |
# Erstellen des Verzeichnisses, falls es nicht existiert
|
| 403 |
if not os.path.exists(directory):
|
|
@@ -420,7 +419,8 @@ def load_splits_and_metadata(directory="chroma/kkg", filename="splits_and_metada
|
|
| 420 |
return pickle.load(f)
|
| 421 |
return None
|
| 422 |
|
| 423 |
-
|
|
|
|
| 424 |
#Laden des Vektorstores - aus den gespeicherten splits
|
| 425 |
def load_vectorstore():
|
| 426 |
splits_and_metadata = load_splits_and_metadata()
|
|
@@ -428,8 +428,9 @@ def load_vectorstore():
|
|
| 428 |
PREPROCESSED_SPLITS, SPLIT_TO_ORIGINAL_MAPPING = splits_and_metadata
|
| 429 |
return document_storage_chroma(PREPROCESSED_SPLITS)
|
| 430 |
return None
|
| 431 |
-
"""
|
| 432 |
|
|
|
|
|
|
|
| 433 |
def save_split_to_original_mapping(mapping, directory="chroma/kkg", mapping_filename="mapping.pkl"):
|
| 434 |
# Erstellen des Verzeichnisses, falls es nicht existiert
|
| 435 |
if not os.path.exists(directory):
|
|
@@ -443,6 +444,7 @@ def save_split_to_original_mapping(mapping, directory="chroma/kkg", mapping_file
|
|
| 443 |
# Hochladen der Mapping-Datei zum Hugging Face Space
|
| 444 |
upload_file_to_huggingface(mapping_filepath, f"{directory}/{mapping_filename}")
|
| 445 |
|
|
|
|
| 446 |
def load_split_to_original_mapping(directory="chroma/kkg", mapping_filename="mapping.pkl"):
|
| 447 |
# Laden des Mappings
|
| 448 |
mapping_filepath = os.path.join(directory, mapping_filename)
|
|
@@ -452,6 +454,15 @@ def load_split_to_original_mapping(directory="chroma/kkg", mapping_filename="map
|
|
| 452 |
return pickle.load(f)
|
| 453 |
return None
|
| 454 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 455 |
def save_vectorstore(vectorstore, directory="chroma/kkg", splits_filename="splits_and_metadata.pkl", vectorstore_filename="vectorstore.pkl"):
|
| 456 |
# Erstellen des Verzeichnisses, falls es nicht existiert
|
| 457 |
if not os.path.exists(directory):
|
|
@@ -485,7 +496,7 @@ def load_vectorstore(directory="chroma/kkg", splits_filename="splits_and_metadat
|
|
| 485 |
vectorstore.documents = documents # Setze die Dokumente im Vektorstore
|
| 486 |
return vectorstore, SPLIT_TO_ORIGINAL_MAPPING
|
| 487 |
return None, None
|
| 488 |
-
|
| 489 |
|
| 490 |
|
| 491 |
|
|
|
|
| 396 |
return vectorstore
|
| 397 |
|
| 398 |
########################################################
|
| 399 |
+
#Splits für den Vektorstore speichern - bzw. laden
|
|
|
|
| 400 |
def save_splits_and_metadata(splits, directory="chroma/kkg", filename="splits_and_metadata.pkl"):
|
| 401 |
# Erstellen des Verzeichnisses, falls es nicht existiert
|
| 402 |
if not os.path.exists(directory):
|
|
|
|
| 419 |
return pickle.load(f)
|
| 420 |
return None
|
| 421 |
|
| 422 |
+
########################################################
|
| 423 |
+
#Vektorstore speichern - bzw. laden
|
| 424 |
#Laden des Vektorstores - aus den gespeicherten splits
|
| 425 |
def load_vectorstore():
|
| 426 |
splits_and_metadata = load_splits_and_metadata()
|
|
|
|
| 428 |
PREPROCESSED_SPLITS, SPLIT_TO_ORIGINAL_MAPPING = splits_and_metadata
|
| 429 |
return document_storage_chroma(PREPROCESSED_SPLITS)
|
| 430 |
return None
|
|
|
|
| 431 |
|
| 432 |
+
#################################
|
| 433 |
+
#das Mapping der orginal-Splits und der preprocessed Splits speichern
|
| 434 |
def save_split_to_original_mapping(mapping, directory="chroma/kkg", mapping_filename="mapping.pkl"):
|
| 435 |
# Erstellen des Verzeichnisses, falls es nicht existiert
|
| 436 |
if not os.path.exists(directory):
|
|
|
|
| 444 |
# Hochladen der Mapping-Datei zum Hugging Face Space
|
| 445 |
upload_file_to_huggingface(mapping_filepath, f"{directory}/{mapping_filename}")
|
| 446 |
|
| 447 |
+
|
| 448 |
def load_split_to_original_mapping(directory="chroma/kkg", mapping_filename="mapping.pkl"):
|
| 449 |
# Laden des Mappings
|
| 450 |
mapping_filepath = os.path.join(directory, mapping_filename)
|
|
|
|
| 454 |
return pickle.load(f)
|
| 455 |
return None
|
| 456 |
|
| 457 |
+
|
| 458 |
+
|
| 459 |
+
|
| 460 |
+
|
| 461 |
+
|
| 462 |
+
|
| 463 |
+
|
| 464 |
+
"""
|
| 465 |
+
|
| 466 |
def save_vectorstore(vectorstore, directory="chroma/kkg", splits_filename="splits_and_metadata.pkl", vectorstore_filename="vectorstore.pkl"):
|
| 467 |
# Erstellen des Verzeichnisses, falls es nicht existiert
|
| 468 |
if not os.path.exists(directory):
|
|
|
|
| 496 |
vectorstore.documents = documents # Setze die Dokumente im Vektorstore
|
| 497 |
return vectorstore, SPLIT_TO_ORIGINAL_MAPPING
|
| 498 |
return None, None
|
| 499 |
+
"""
|
| 500 |
|
| 501 |
|
| 502 |
|