add corpus embeddings

creating_corpus_embeddings.py

@@ -0,0 +1,26 @@
+from safetensors.torch import save_file
+from sentence_transformers import SentenceTransformer
+import json
+
+
+# load taxonomy
+with open("taxonomy_processed_v3.json", "r") as fp:
+    taxonomy = json.load(fp)
+
+# load model
+model = SentenceTransformer(
+    model_name_or_path="and-effect/musterdatenkatalog_clf",
+    device="cpu",
+    use_auth_token=True,
+)
+
+# get corpus labels
+corpus_labels = [el["group"] + " - " + el["label"] for el in taxonomy]
+
+# get corpus embeddings
+corpus_embeddings = model.encode(corpus_labels, convert_to_tensor=True)
+
+# save corpus embeddings
+tensors = {"corpus_embeddings": corpus_embeddings}
+
+save_file(tensors, "corpus_embeddings.pt")