zero-shot benchmark

research_files/benchmark/segmentation_benchmark/zero_shot_transfer.py

@@ -81,6 +81,7 @@ def zero_shot_proposed(
     rho, _ = spearmanr(y, y_hat)
 
     print(f"Average Spearman correlation: {rho:.4f}")
+    return rho
 # - x - x - x - x - x - x - x - x - x - x - x - x - x - x - #
 #                        END OF FILE                        #
 # - x - x - x - x - x - x - x - x - x - x - x - x - x - x - #

research_files/zero_shot_results.json

@@ -0,0 +1,8 @@
+{
+    "Alverciito/wikipedia_segmentation": 0.13476829801490514,
+    "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2": 0.8007057781768085,
+    "hiiamsid/sentence_similarity_spanish_es": 0.8126300355781378,
+    "google-bert/bert-base-multilingual-cased": 0.5917908219072123,
+    "sentence-transformers/LaBSE": 0.7317496155094046,
+    "FacebookAI/xlm-roberta-base": 0.5289939928786738
+}

research_files/zero_shot_tranfer_experiment.py

@@ -5,17 +5,31 @@
 #                                                           #
 # - x - x - x - x - x - x - x - x - x - x - x - x - x - x - #
 # Import statements:
-# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - #
-#                         START OF FILE                             #
-# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - #
 import os
+import json
 from benchmark.segmentation_benchmark.zero_shot_transfer import zero_shot_proposed
 
+MODELS = [
+    "Alverciito/wikipedia_segmentation",
+    "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2",  # Baseline Multilingual - sBERT
+    'hiiamsid/sentence_similarity_spanish_es',  # Spanish similarity - sBERT
+    # "jaimevera1107/all-MiniLM-L6-v2-similarity-es",  # Spanish similarity - sBERT
+    "google-bert/bert-base-multilingual-cased",  # mBERT (google)
+    "sentence-transformers/LaBSE",  # LaBSE (google)
+    "FacebookAI/xlm-roberta-base"  # XLM-R (facebook)
+]
+
 __file_path__ = os.path.dirname(__file__)
 
 if __name__ == '__main__':
-    # zero_shot_proposed("hiiamsid/sentence_similarity_spanish_es", "nflechas/semantic_sentence_similarity_ES")
-    zero_shot_proposed("Alverciito/wikipedia_segmentation", "nflechas/semantic_sentence_similarity_ES")
+    results = dict()
+    for model in MODELS:
+        rho = zero_shot_proposed(model, "nflechas/semantic_sentence_similarity_ES")
+        results[model] = rho
+
+        with open(os.path.join(__file_path__, 'zero_shot_results.json'), 'w', encoding='utf-8') as f:
+            json.dump(results, f, indent=4, ensure_ascii=False)
+
 # - x - x - x - x - x - x - x - x - x - x - x - x - x - x - #
 #                        END OF FILE                        #
 # - x - x - x - x - x - x - x - x - x - x - x - x - x - x - #