Upload 130 files

.gitattributes

@@ -32,3 +32,7 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+qa_output/nq-test/results.jsonl filter=lfs diff=lfs merge=lfs -text
+qa_output/squad1-test/results.jsonl filter=lfs diff=lfs merge=lfs -text
+qa_output/trivia-test/results.jsonl filter=lfs diff=lfs merge=lfs -text
+qa_output/webq-test/results.jsonl filter=lfs diff=lfs merge=lfs -text

beir_output/arguana.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.16501,0.27175,0.31815,0.37955,0.4119,0.43478,0.44184,0.44581,0.44949
+map,0.16501,0.24526,0.2709,0.29624,0.30539,0.30919,0.30982,0.31002,0.31011
+precision,0.16501,0.11617,0.09232,0.06515,0.03887,0.01782,0.00935,0.00482,0.00099
+recall,0.16501,0.34851,0.46159,0.65149,0.77738,0.89118,0.93457,0.96302,0.99431
+mrr,0.1707,0.24798,0.27291,0.29858,0.30765,0.31146,0.31208,0.31228,0.31237
+recall_cap,0.1707,0.34993,0.46017,0.6522,0.77738,0.89189,0.93457,0.96302,0.99431
+hole,0.67568,0.68753,0.68834,0.67817,0.7181,0.79438,0.83069,0.85393,0.87081

beir_output/arguana.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "arguana",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.16501,
+            "NDCG@3": 0.27175,
+            "NDCG@5": 0.31815,
+            "NDCG@10": 0.37955,
+            "NDCG@20": 0.4119,
+            "NDCG@50": 0.43478,
+            "NDCG@100": 0.44184,
+            "NDCG@200": 0.44581,
+            "NDCG@1000": 0.44949
+        },
+        "map": {
+            "MAP@1": 0.16501,
+            "MAP@3": 0.24526,
+            "MAP@5": 0.2709,
+            "MAP@10": 0.29624,
+            "MAP@20": 0.30539,
+            "MAP@50": 0.30919,
+            "MAP@100": 0.30982,
+            "MAP@200": 0.31002,
+            "MAP@1000": 0.31011
+        },
+        "precision": {
+            "P@1": 0.16501,
+            "P@3": 0.11617,
+            "P@5": 0.09232,
+            "P@10": 0.06515,
+            "P@20": 0.03887,
+            "P@50": 0.01782,
+            "P@100": 0.00935,
+            "P@200": 0.00482,
+            "P@1000": 0.00099
+        },
+        "recall": {
+            "Recall@1": 0.16501,
+            "Recall@3": 0.34851,
+            "Recall@5": 0.46159,
+            "Recall@10": 0.65149,
+            "Recall@20": 0.77738,
+            "Recall@50": 0.89118,
+            "Recall@100": 0.93457,
+            "Recall@200": 0.96302,
+            "Recall@1000": 0.99431
+        },
+        "mrr": {
+            "MRR@1": 0.1707,
+            "MRR@3": 0.24798,
+            "MRR@5": 0.27291,
+            "MRR@10": 0.29858,
+            "MRR@20": 0.30765,
+            "MRR@50": 0.31146,
+            "MRR@100": 0.31208,
+            "MRR@200": 0.31228,
+            "MRR@1000": 0.31237
+        },
+        "recall_cap": {
+            "R_cap@1": 0.1707,
+            "R_cap@3": 0.34993,
+            "R_cap@5": 0.46017,
+            "R_cap@10": 0.6522,
+            "R_cap@20": 0.77738,
+            "R_cap@50": 0.89189,
+            "R_cap@100": 0.93457,
+            "R_cap@200": 0.96302,
+            "R_cap@1000": 0.99431
+        },
+        "hole": {
+            "Hole@1": 0.67568,
+            "Hole@3": 0.68753,
+            "Hole@5": 0.68834,
+            "Hole@10": 0.67817,
+            "Hole@20": 0.7181,
+            "Hole@50": 0.79438,
+            "Hole@100": 0.83069,
+            "Hole@200": 0.85393,
+            "Hole@1000": 0.87081
+        }
+    }
+}

beir_output/beir.json

@@ -0,0 +1,26 @@
+{
+    "eval_beir-msmarco_ndcg@10": 0.23486,
+    "eval_beir-msmarco_recall@10": 0.40681,
+    "eval_beir-msmarco_recall@20": 0.50885,
+    "eval_beir-msmarco_recall@100": 0.7191,
+    "eval_beir-dbpedia-entity_ndcg@10": 0.32317,
+    "eval_beir-dbpedia-entity_recall@10": 0.21027,
+    "eval_beir-dbpedia-entity_recall@20": 0.27414,
+    "eval_beir-dbpedia-entity_recall@100": 0.46921,
+    "eval_beir-fever_ndcg@10": 0.6139,
+    "eval_beir-fever_recall@10": 0.79606,
+    "eval_beir-fever_recall@20": 0.85168,
+    "eval_beir-fever_recall@100": 0.91454,
+    "eval_beir-climate-fever_ndcg@10": 0.17477,
+    "eval_beir-climate-fever_recall@10": 0.21757,
+    "eval_beir-climate-fever_recall@20": 0.27878,
+    "eval_beir-climate-fever_recall@100": 0.47127,
+    "eval_beir-hotpotqa_ndcg@10": 0.53618,
+    "eval_beir-hotpotqa_recall@10": 0.57016,
+    "eval_beir-hotpotqa_recall@20": 0.61931,
+    "eval_beir-hotpotqa_recall@100": 0.72073,
+    "eval_beir-avg_ndcg@10": 0.376576,
+    "eval_beir-avg_recall@10": 0.440174,
+    "eval_beir-avg_recall@20": 0.506552,
+    "eval_beir-avg_recall@100": 0.6589700000000001
+}

beir_output/climate-fever.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.15375,0.14109,0.15189,0.17477,0.19587,0.22467,0.2446,0.2594,0.28801
+map,0.07286,0.1021,0.11124,0.12081,0.12703,0.13232,0.13455,0.13568,0.13668
+precision,0.15375,0.10337,0.07883,0.05388,0.03577,0.02059,0.01296,0.00766,0.0021
+recall,0.07286,0.13249,0.16442,0.21757,0.27878,0.38271,0.47127,0.54594,0.7195
+mrr,0.15375,0.21107,0.22436,0.23645,0.24293,0.24785,0.24919,0.24952,0.24979
+recall_cap,0.15375,0.14104,0.16442,0.21757,0.27878,0.38271,0.47127,0.54594,0.7195
+hole,0.6215,0.676,0.71362,0.7514,0.78668,0.82293,0.84855,0.87267,0.92384

beir_output/climate-fever.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "climate-fever",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.15375,
+            "NDCG@3": 0.14109,
+            "NDCG@5": 0.15189,
+            "NDCG@10": 0.17477,
+            "NDCG@20": 0.19587,
+            "NDCG@50": 0.22467,
+            "NDCG@100": 0.2446,
+            "NDCG@200": 0.2594,
+            "NDCG@1000": 0.28801
+        },
+        "map": {
+            "MAP@1": 0.07286,
+            "MAP@3": 0.1021,
+            "MAP@5": 0.11124,
+            "MAP@10": 0.12081,
+            "MAP@20": 0.12703,
+            "MAP@50": 0.13232,
+            "MAP@100": 0.13455,
+            "MAP@200": 0.13568,
+            "MAP@1000": 0.13668
+        },
+        "precision": {
+            "P@1": 0.15375,
+            "P@3": 0.10337,
+            "P@5": 0.07883,
+            "P@10": 0.05388,
+            "P@20": 0.03577,
+            "P@50": 0.02059,
+            "P@100": 0.01296,
+            "P@200": 0.00766,
+            "P@1000": 0.0021
+        },
+        "recall": {
+            "Recall@1": 0.07286,
+            "Recall@3": 0.13249,
+            "Recall@5": 0.16442,
+            "Recall@10": 0.21757,
+            "Recall@20": 0.27878,
+            "Recall@50": 0.38271,
+            "Recall@100": 0.47127,
+            "Recall@200": 0.54594,
+            "Recall@1000": 0.7195
+        },
+        "mrr": {
+            "MRR@1": 0.15375,
+            "MRR@3": 0.21107,
+            "MRR@5": 0.22436,
+            "MRR@10": 0.23645,
+            "MRR@20": 0.24293,
+            "MRR@50": 0.24785,
+            "MRR@100": 0.24919,
+            "MRR@200": 0.24952,
+            "MRR@1000": 0.24979
+        },
+        "recall_cap": {
+            "R_cap@1": 0.15375,
+            "R_cap@3": 0.14104,
+            "R_cap@5": 0.16442,
+            "R_cap@10": 0.21757,
+            "R_cap@20": 0.27878,
+            "R_cap@50": 0.38271,
+            "R_cap@100": 0.47127,
+            "R_cap@200": 0.54594,
+            "R_cap@1000": 0.7195
+        },
+        "hole": {
+            "Hole@1": 0.6215,
+            "Hole@3": 0.676,
+            "Hole@5": 0.71362,
+            "Hole@10": 0.7514,
+            "Hole@20": 0.78668,
+            "Hole@50": 0.82293,
+            "Hole@100": 0.84855,
+            "Hole@200": 0.87267,
+            "Hole@1000": 0.92384
+        }
+    }
+}

beir_output/cqadupstack.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.18406,0.2229525,0.24138916666666663,0.264705,0.2836966666666667,0.3036575,0.317455,0.3279316666666667,0.3471208333333333
+map,0.15620583333333335,0.19959000000000002,0.21138416666666668,0.22192916666666665,0.22766583333333335,0.23130833333333337,0.2327866666666667,0.2334733333333333,0.23405083333333332
+precision,0.18406,0.10411250000000001,0.0760425,0.04850833333333333,0.0298675,0.015099166666666665,0.008941666666666665,0.005136666666666667,0.0013391666666666667
+recall,0.15620583333333335,0.24788249999999998,0.2942608333333333,0.3632225,0.4335508333333334,0.5252308333333333,0.6014158333333334,0.6680558333333332,0.8154633333333333
+mrr,0.18406,0.23019749999999997,0.24204666666666663,0.25234,0.257585,0.2605608333333334,0.26161333333333336,0.26206250000000003,0.26239833333333334
+recall_cap,0.18406,0.2531541666666666,0.2966533333333334,0.3642716666666666,0.43398416666666667,0.5253808333333334,0.6014775,0.6680633333333333,0.8154633333333333
+hole,0.7429125,0.8188533333333333,0.8458958333333334,0.8714099999999999,0.8905433333333334,0.9064025,0.9140950000000001,0.9195925000000003,0.927175

beir_output/cqadupstack.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "cqadupstack",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.18406,
+            "NDCG@3": 0.2229525,
+            "NDCG@5": 0.24138916666666663,
+            "NDCG@10": 0.264705,
+            "NDCG@20": 0.2836966666666667,
+            "NDCG@50": 0.3036575,
+            "NDCG@100": 0.317455,
+            "NDCG@200": 0.3279316666666667,
+            "NDCG@1000": 0.3471208333333333
+        },
+        "map": {
+            "MAP@1": 0.15620583333333335,
+            "MAP@3": 0.19959000000000002,
+            "MAP@5": 0.21138416666666668,
+            "MAP@10": 0.22192916666666665,
+            "MAP@20": 0.22766583333333335,
+            "MAP@50": 0.23130833333333337,
+            "MAP@100": 0.2327866666666667,
+            "MAP@200": 0.2334733333333333,
+            "MAP@1000": 0.23405083333333332
+        },
+        "precision": {
+            "P@1": 0.18406,
+            "P@3": 0.10411250000000001,
+            "P@5": 0.0760425,
+            "P@10": 0.04850833333333333,
+            "P@20": 0.0298675,
+            "P@50": 0.015099166666666665,
+            "P@100": 0.008941666666666665,
+            "P@200": 0.005136666666666667,
+            "P@1000": 0.0013391666666666667
+        },
+        "recall": {
+            "Recall@1": 0.15620583333333335,
+            "Recall@3": 0.24788249999999998,
+            "Recall@5": 0.2942608333333333,
+            "Recall@10": 0.3632225,
+            "Recall@20": 0.4335508333333334,
+            "Recall@50": 0.5252308333333333,
+            "Recall@100": 0.6014158333333334,
+            "Recall@200": 0.6680558333333332,
+            "Recall@1000": 0.8154633333333333
+        },
+        "mrr": {
+            "MRR@1": 0.18406,
+            "MRR@3": 0.23019749999999997,
+            "MRR@5": 0.24204666666666663,
+            "MRR@10": 0.25234,
+            "MRR@20": 0.257585,
+            "MRR@50": 0.2605608333333334,
+            "MRR@100": 0.26161333333333336,
+            "MRR@200": 0.26206250000000003,
+            "MRR@1000": 0.26239833333333334
+        },
+        "recall_cap": {
+            "R_cap@1": 0.18406,
+            "R_cap@3": 0.2531541666666666,
+            "R_cap@5": 0.2966533333333334,
+            "R_cap@10": 0.3642716666666666,
+            "R_cap@20": 0.43398416666666667,
+            "R_cap@50": 0.5253808333333334,
+            "R_cap@100": 0.6014775,
+            "R_cap@200": 0.6680633333333333,
+            "R_cap@1000": 0.8154633333333333
+        },
+        "hole": {
+            "Hole@1": 0.7429125,
+            "Hole@3": 0.8188533333333333,
+            "Hole@5": 0.8458958333333334,
+            "Hole@10": 0.8714099999999999,
+            "Hole@20": 0.8905433333333334,
+            "Hole@50": 0.9064025,
+            "Hole@100": 0.9140950000000001,
+            "Hole@200": 0.9195925000000003,
+            "Hole@1000": 0.927175
+        }
+    }
+}

beir_output/dbpedia-entity.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.41,0.34802,0.33273,0.32317,0.32189,0.3495,0.37969,0.4059,0.45673
+map,0.06969,0.10807,0.12589,0.14918,0.17423,0.2035,0.21665,0.22394,0.23158
+precision,0.53,0.38667,0.33,0.2615,0.2005,0.13365,0.08973,0.05703,0.01926
+recall,0.06969,0.12522,0.15947,0.21027,0.27414,0.38471,0.46921,0.54679,0.72284
+mrr,0.53,0.61083,0.62696,0.63464,0.63834,0.64049,0.64077,0.64079,0.64085
+recall_cap,0.53,0.40708,0.37138,0.34473,0.33686,0.3967,0.47456,0.54983,0.72347
+hole,0.285,0.37583,0.4335,0.51325,0.5995,0.71235,0.78725,0.84906,0.9337

beir_output/dbpedia-entity.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "dbpedia-entity",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.41,
+            "NDCG@3": 0.34802,
+            "NDCG@5": 0.33273,
+            "NDCG@10": 0.32317,
+            "NDCG@20": 0.32189,
+            "NDCG@50": 0.3495,
+            "NDCG@100": 0.37969,
+            "NDCG@200": 0.4059,
+            "NDCG@1000": 0.45673
+        },
+        "map": {
+            "MAP@1": 0.06969,
+            "MAP@3": 0.10807,
+            "MAP@5": 0.12589,
+            "MAP@10": 0.14918,
+            "MAP@20": 0.17423,
+            "MAP@50": 0.2035,
+            "MAP@100": 0.21665,
+            "MAP@200": 0.22394,
+            "MAP@1000": 0.23158
+        },
+        "precision": {
+            "P@1": 0.53,
+            "P@3": 0.38667,
+            "P@5": 0.33,
+            "P@10": 0.2615,
+            "P@20": 0.2005,
+            "P@50": 0.13365,
+            "P@100": 0.08973,
+            "P@200": 0.05703,
+            "P@1000": 0.01926
+        },
+        "recall": {
+            "Recall@1": 0.06969,
+            "Recall@3": 0.12522,
+            "Recall@5": 0.15947,
+            "Recall@10": 0.21027,
+            "Recall@20": 0.27414,
+            "Recall@50": 0.38471,
+            "Recall@100": 0.46921,
+            "Recall@200": 0.54679,
+            "Recall@1000": 0.72284
+        },
+        "mrr": {
+            "MRR@1": 0.53,
+            "MRR@3": 0.61083,
+            "MRR@5": 0.62696,
+            "MRR@10": 0.63464,
+            "MRR@20": 0.63834,
+            "MRR@50": 0.64049,
+            "MRR@100": 0.64077,
+            "MRR@200": 0.64079,
+            "MRR@1000": 0.64085
+        },
+        "recall_cap": {
+            "R_cap@1": 0.53,
+            "R_cap@3": 0.40708,
+            "R_cap@5": 0.37138,
+            "R_cap@10": 0.34473,
+            "R_cap@20": 0.33686,
+            "R_cap@50": 0.3967,
+            "R_cap@100": 0.47456,
+            "R_cap@200": 0.54983,
+            "R_cap@1000": 0.72347
+        },
+        "hole": {
+            "Hole@1": 0.285,
+            "Hole@3": 0.37583,
+            "Hole@5": 0.4335,
+            "Hole@10": 0.51325,
+            "Hole@20": 0.5995,
+            "Hole@50": 0.71235,
+            "Hole@100": 0.78725,
+            "Hole@200": 0.84906,
+            "Hole@1000": 0.9337
+        }
+    }
+}

beir_output/fever.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.44119,0.55524,0.58732,0.6139,0.62866,0.63731,0.64113,0.6432,0.6469
+map,0.41174,0.51463,0.53292,0.54442,0.54871,0.55026,0.55064,0.55078,0.55089
+precision,0.44119,0.22957,0.15518,0.08672,0.04665,0.0197,0.01014,0.00518,0.00108
+recall,0.41174,0.63935,0.71639,0.79606,0.85168,0.89261,0.91454,0.92749,0.95598
+mrr,0.44119,0.54768,0.56651,0.57755,0.58159,0.58286,0.58316,0.58322,0.58328
+recall_cap,0.44119,0.64041,0.71664,0.79607,0.85168,0.89261,0.91454,0.92749,0.95598
+hole,0.5312,0.74257,0.81869,0.89305,0.93924,0.97161,0.98398,0.99077,0.99684

beir_output/fever.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "fever",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.44119,
+            "NDCG@3": 0.55524,
+            "NDCG@5": 0.58732,
+            "NDCG@10": 0.6139,
+            "NDCG@20": 0.62866,
+            "NDCG@50": 0.63731,
+            "NDCG@100": 0.64113,
+            "NDCG@200": 0.6432,
+            "NDCG@1000": 0.6469
+        },
+        "map": {
+            "MAP@1": 0.41174,
+            "MAP@3": 0.51463,
+            "MAP@5": 0.53292,
+            "MAP@10": 0.54442,
+            "MAP@20": 0.54871,
+            "MAP@50": 0.55026,
+            "MAP@100": 0.55064,
+            "MAP@200": 0.55078,
+            "MAP@1000": 0.55089
+        },
+        "precision": {
+            "P@1": 0.44119,
+            "P@3": 0.22957,
+            "P@5": 0.15518,
+            "P@10": 0.08672,
+            "P@20": 0.04665,
+            "P@50": 0.0197,
+            "P@100": 0.01014,
+            "P@200": 0.00518,
+            "P@1000": 0.00108
+        },
+        "recall": {
+            "Recall@1": 0.41174,
+            "Recall@3": 0.63935,
+            "Recall@5": 0.71639,
+            "Recall@10": 0.79606,
+            "Recall@20": 0.85168,
+            "Recall@50": 0.89261,
+            "Recall@100": 0.91454,
+            "Recall@200": 0.92749,
+            "Recall@1000": 0.95598
+        },
+        "mrr": {
+            "MRR@1": 0.44119,
+            "MRR@3": 0.54768,
+            "MRR@5": 0.56651,
+            "MRR@10": 0.57755,
+            "MRR@20": 0.58159,
+            "MRR@50": 0.58286,
+            "MRR@100": 0.58316,
+            "MRR@200": 0.58322,
+            "MRR@1000": 0.58328
+        },
+        "recall_cap": {
+            "R_cap@1": 0.44119,
+            "R_cap@3": 0.64041,
+            "R_cap@5": 0.71664,
+            "R_cap@10": 0.79607,
+            "R_cap@20": 0.85168,
+            "R_cap@50": 0.89261,
+            "R_cap@100": 0.91454,
+            "R_cap@200": 0.92749,
+            "R_cap@1000": 0.95598
+        },
+        "hole": {
+            "Hole@1": 0.5312,
+            "Hole@3": 0.74257,
+            "Hole@5": 0.81869,
+            "Hole@10": 0.89305,
+            "Hole@20": 0.93924,
+            "Hole@50": 0.97161,
+            "Hole@100": 0.98398,
+            "Hole@200": 0.99077,
+            "Hole@1000": 0.99684
+        }
+    }
+}

beir_output/fiqa.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.23302,0.2207,0.22989,0.25245,0.27483,0.30454,0.32168,0.33558,0.36481
+map,0.1186,0.16754,0.18035,0.19302,0.20042,0.20645,0.20851,0.2096,0.21068
+precision,0.23302,0.1466,0.10772,0.07068,0.04414,0.02352,0.01384,0.00802,0.00216
+recall,0.1186,0.20366,0.24624,0.31073,0.37975,0.49631,0.57848,0.65152,0.84222
+mrr,0.23302,0.28627,0.29992,0.3112,0.31809,0.32196,0.32302,0.32337,0.32368
+recall_cap,0.23302,0.22659,0.25239,0.31126,0.37975,0.49631,0.57848,0.65152,0.84222
+hole,0.71296,0.79681,0.83333,0.8679,0.89367,0.91238,0.92168,0.92719,0.93598

beir_output/fiqa.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "fiqa",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.23302,
+            "NDCG@3": 0.2207,
+            "NDCG@5": 0.22989,
+            "NDCG@10": 0.25245,
+            "NDCG@20": 0.27483,
+            "NDCG@50": 0.30454,
+            "NDCG@100": 0.32168,
+            "NDCG@200": 0.33558,
+            "NDCG@1000": 0.36481
+        },
+        "map": {
+            "MAP@1": 0.1186,
+            "MAP@3": 0.16754,
+            "MAP@5": 0.18035,
+            "MAP@10": 0.19302,
+            "MAP@20": 0.20042,
+            "MAP@50": 0.20645,
+            "MAP@100": 0.20851,
+            "MAP@200": 0.2096,
+            "MAP@1000": 0.21068
+        },
+        "precision": {
+            "P@1": 0.23302,
+            "P@3": 0.1466,
+            "P@5": 0.10772,
+            "P@10": 0.07068,
+            "P@20": 0.04414,
+            "P@50": 0.02352,
+            "P@100": 0.01384,
+            "P@200": 0.00802,
+            "P@1000": 0.00216
+        },
+        "recall": {
+            "Recall@1": 0.1186,
+            "Recall@3": 0.20366,
+            "Recall@5": 0.24624,
+            "Recall@10": 0.31073,
+            "Recall@20": 0.37975,
+            "Recall@50": 0.49631,
+            "Recall@100": 0.57848,
+            "Recall@200": 0.65152,
+            "Recall@1000": 0.84222
+        },
+        "mrr": {
+            "MRR@1": 0.23302,
+            "MRR@3": 0.28627,
+            "MRR@5": 0.29992,
+            "MRR@10": 0.3112,
+            "MRR@20": 0.31809,
+            "MRR@50": 0.32196,
+            "MRR@100": 0.32302,
+            "MRR@200": 0.32337,
+            "MRR@1000": 0.32368
+        },
+        "recall_cap": {
+            "R_cap@1": 0.23302,
+            "R_cap@3": 0.22659,
+            "R_cap@5": 0.25239,
+            "R_cap@10": 0.31126,
+            "R_cap@20": 0.37975,
+            "R_cap@50": 0.49631,
+            "R_cap@100": 0.57848,
+            "R_cap@200": 0.65152,
+            "R_cap@1000": 0.84222
+        },
+        "hole": {
+            "Hole@1": 0.71296,
+            "Hole@3": 0.79681,
+            "Hole@5": 0.83333,
+            "Hole@10": 0.8679,
+            "Hole@20": 0.89367,
+            "Hole@50": 0.91238,
+            "Hole@100": 0.92168,
+            "Hole@200": 0.92719,
+            "Hole@1000": 0.93598
+        }
+    }
+}

beir_output/hotpotqa.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.61621,0.48644,0.51182,0.53618,0.55146,0.56649,0.57437,0.58079,0.59128
+map,0.3081,0.40744,0.42517,0.43827,0.44401,0.4474,0.4484,0.44888,0.44924
+precision,0.61621,0.30587,0.20365,0.11403,0.06193,0.02724,0.01441,0.00758,0.00167
+recall,0.3081,0.45881,0.50912,0.57016,0.61931,0.68109,0.72073,0.75814,0.83282
+mrr,0.61621,0.68578,0.6962,0.70307,0.70536,0.70645,0.70671,0.70682,0.70687
+recall_cap,0.61621,0.45881,0.50912,0.57016,0.61931,0.68109,0.72073,0.75814,0.83282
+hole,0.37664,0.68283,0.78439,0.87353,0.92629,0.96172,0.97511,0.98258,0.99002

beir_output/hotpotqa.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "hotpotqa",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.61621,
+            "NDCG@3": 0.48644,
+            "NDCG@5": 0.51182,
+            "NDCG@10": 0.53618,
+            "NDCG@20": 0.55146,
+            "NDCG@50": 0.56649,
+            "NDCG@100": 0.57437,
+            "NDCG@200": 0.58079,
+            "NDCG@1000": 0.59128
+        },
+        "map": {
+            "MAP@1": 0.3081,
+            "MAP@3": 0.40744,
+            "MAP@5": 0.42517,
+            "MAP@10": 0.43827,
+            "MAP@20": 0.44401,
+            "MAP@50": 0.4474,
+            "MAP@100": 0.4484,
+            "MAP@200": 0.44888,
+            "MAP@1000": 0.44924
+        },
+        "precision": {
+            "P@1": 0.61621,
+            "P@3": 0.30587,
+            "P@5": 0.20365,
+            "P@10": 0.11403,
+            "P@20": 0.06193,
+            "P@50": 0.02724,
+            "P@100": 0.01441,
+            "P@200": 0.00758,
+            "P@1000": 0.00167
+        },
+        "recall": {
+            "Recall@1": 0.3081,
+            "Recall@3": 0.45881,
+            "Recall@5": 0.50912,
+            "Recall@10": 0.57016,
+            "Recall@20": 0.61931,
+            "Recall@50": 0.68109,
+            "Recall@100": 0.72073,
+            "Recall@200": 0.75814,
+            "Recall@1000": 0.83282
+        },
+        "mrr": {
+            "MRR@1": 0.61621,
+            "MRR@3": 0.68578,
+            "MRR@5": 0.6962,
+            "MRR@10": 0.70307,
+            "MRR@20": 0.70536,
+            "MRR@50": 0.70645,
+            "MRR@100": 0.70671,
+            "MRR@200": 0.70682,
+            "MRR@1000": 0.70687
+        },
+        "recall_cap": {
+            "R_cap@1": 0.61621,
+            "R_cap@3": 0.45881,
+            "R_cap@5": 0.50912,
+            "R_cap@10": 0.57016,
+            "R_cap@20": 0.61931,
+            "R_cap@50": 0.68109,
+            "R_cap@100": 0.72073,
+            "R_cap@200": 0.75814,
+            "R_cap@1000": 0.83282
+        },
+        "hole": {
+            "Hole@1": 0.37664,
+            "Hole@3": 0.68283,
+            "Hole@5": 0.78439,
+            "Hole@10": 0.87353,
+            "Hole@20": 0.92629,
+            "Hole@50": 0.96172,
+            "Hole@100": 0.97511,
+            "Hole@200": 0.98258,
+            "Hole@1000": 0.99002
+        }
+    }
+}

beir_output/msmarco.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.09756,0.16557,0.19803,0.23486,0.26094,0.28617,0.30026,0.31119,0.32567
+map,0.09506,0.14688,0.16504,0.18046,0.18762,0.19176,0.19304,0.19362,0.19398
+precision,0.09756,0.0745,0.06089,0.04219,0.02649,0.01326,0.00754,0.00419,0.00097
+recall,0.09506,0.2164,0.29451,0.40681,0.50885,0.63387,0.7191,0.79531,0.91741
+mrr,0.09756,0.15036,0.16834,0.1836,0.19079,0.19482,0.19605,0.19661,0.19693
+recall_cap,0.09756,0.2164,0.29436,0.40681,0.50885,0.63387,0.7191,0.79531,0.91741
+hole,0.901,0.92426,0.93779,0.9566,0.97236,0.98568,0.99139,0.99473,0.99799

beir_output/msmarco.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "msmarco",
+    "split": "dev",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.09756,
+            "NDCG@3": 0.16557,
+            "NDCG@5": 0.19803,
+            "NDCG@10": 0.23486,
+            "NDCG@20": 0.26094,
+            "NDCG@50": 0.28617,
+            "NDCG@100": 0.30026,
+            "NDCG@200": 0.31119,
+            "NDCG@1000": 0.32567
+        },
+        "map": {
+            "MAP@1": 0.09506,
+            "MAP@3": 0.14688,
+            "MAP@5": 0.16504,
+            "MAP@10": 0.18046,
+            "MAP@20": 0.18762,
+            "MAP@50": 0.19176,
+            "MAP@100": 0.19304,
+            "MAP@200": 0.19362,
+            "MAP@1000": 0.19398
+        },
+        "precision": {
+            "P@1": 0.09756,
+            "P@3": 0.0745,
+            "P@5": 0.06089,
+            "P@10": 0.04219,
+            "P@20": 0.02649,
+            "P@50": 0.01326,
+            "P@100": 0.00754,
+            "P@200": 0.00419,
+            "P@1000": 0.00097
+        },
+        "recall": {
+            "Recall@1": 0.09506,
+            "Recall@3": 0.2164,
+            "Recall@5": 0.29451,
+            "Recall@10": 0.40681,
+            "Recall@20": 0.50885,
+            "Recall@50": 0.63387,
+            "Recall@100": 0.7191,
+            "Recall@200": 0.79531,
+            "Recall@1000": 0.91741
+        },
+        "mrr": {
+            "MRR@1": 0.09756,
+            "MRR@3": 0.15036,
+            "MRR@5": 0.16834,
+            "MRR@10": 0.1836,
+            "MRR@20": 0.19079,
+            "MRR@50": 0.19482,
+            "MRR@100": 0.19605,
+            "MRR@200": 0.19661,
+            "MRR@1000": 0.19693
+        },
+        "recall_cap": {
+            "R_cap@1": 0.09756,
+            "R_cap@3": 0.2164,
+            "R_cap@5": 0.29436,
+            "R_cap@10": 0.40681,
+            "R_cap@20": 0.50885,
+            "R_cap@50": 0.63387,
+            "R_cap@100": 0.7191,
+            "R_cap@200": 0.79531,
+            "R_cap@1000": 0.91741
+        },
+        "hole": {
+            "Hole@1": 0.901,
+            "Hole@3": 0.92426,
+            "Hole@5": 0.93779,
+            "Hole@10": 0.9566,
+            "Hole@20": 0.97236,
+            "Hole@50": 0.98568,
+            "Hole@100": 0.99139,
+            "Hole@200": 0.99473,
+            "Hole@1000": 0.99799
+        }
+    }
+}

beir_output/nfcorpus.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.36068,0.34345,0.33229,0.30262,0.27932,0.27176,0.27884,0.29539,0.36641
+map,0.03847,0.07726,0.09253,0.10631,0.11761,0.12837,0.13406,0.13904,0.14774
+precision,0.3808,0.33333,0.29907,0.22786,0.16594,0.10854,0.07372,0.04961,0.02026
+recall,0.03847,0.09245,0.1199,0.15211,0.18187,0.24018,0.28685,0.35578,0.60393
+mrr,0.387,0.46078,0.47843,0.48676,0.48949,0.49217,0.49262,0.49294,0.49317
+recall_cap,0.387,0.35191,0.33359,0.29074,0.26102,0.27139,0.30159,0.36072,0.60393
+hole,0.08359,0.08153,0.08607,0.10588,0.11254,0.12297,0.12409,0.12789,0.13433

beir_output/nfcorpus.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "nfcorpus",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.36068,
+            "NDCG@3": 0.34345,
+            "NDCG@5": 0.33229,
+            "NDCG@10": 0.30262,
+            "NDCG@20": 0.27932,
+            "NDCG@50": 0.27176,
+            "NDCG@100": 0.27884,
+            "NDCG@200": 0.29539,
+            "NDCG@1000": 0.36641
+        },
+        "map": {
+            "MAP@1": 0.03847,
+            "MAP@3": 0.07726,
+            "MAP@5": 0.09253,
+            "MAP@10": 0.10631,
+            "MAP@20": 0.11761,
+            "MAP@50": 0.12837,
+            "MAP@100": 0.13406,
+            "MAP@200": 0.13904,
+            "MAP@1000": 0.14774
+        },
+        "precision": {
+            "P@1": 0.3808,
+            "P@3": 0.33333,
+            "P@5": 0.29907,
+            "P@10": 0.22786,
+            "P@20": 0.16594,
+            "P@50": 0.10854,
+            "P@100": 0.07372,
+            "P@200": 0.04961,
+            "P@1000": 0.02026
+        },
+        "recall": {
+            "Recall@1": 0.03847,
+            "Recall@3": 0.09245,
+            "Recall@5": 0.1199,
+            "Recall@10": 0.15211,
+            "Recall@20": 0.18187,
+            "Recall@50": 0.24018,
+            "Recall@100": 0.28685,
+            "Recall@200": 0.35578,
+            "Recall@1000": 0.60393
+        },
+        "mrr": {
+            "MRR@1": 0.387,
+            "MRR@3": 0.46078,
+            "MRR@5": 0.47843,
+            "MRR@10": 0.48676,
+            "MRR@20": 0.48949,
+            "MRR@50": 0.49217,
+            "MRR@100": 0.49262,
+            "MRR@200": 0.49294,
+            "MRR@1000": 0.49317
+        },
+        "recall_cap": {
+            "R_cap@1": 0.387,
+            "R_cap@3": 0.35191,
+            "R_cap@5": 0.33359,
+            "R_cap@10": 0.29074,
+            "R_cap@20": 0.26102,
+            "R_cap@50": 0.27139,
+            "R_cap@100": 0.30159,
+            "R_cap@200": 0.36072,
+            "R_cap@1000": 0.60393
+        },
+        "hole": {
+            "Hole@1": 0.08359,
+            "Hole@3": 0.08153,
+            "Hole@5": 0.08607,
+            "Hole@10": 0.10588,
+            "Hole@20": 0.11254,
+            "Hole@50": 0.12297,
+            "Hole@100": 0.12409,
+            "Hole@200": 0.12789,
+            "Hole@1000": 0.13433
+        }
+    }
+}

beir_output/nq.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.13789,0.20975,0.24822,0.28918,0.32125,0.3507,0.36448,0.37301,0.38303
+map,0.12029,0.18428,0.20611,0.22412,0.23357,0.23878,0.24015,0.24064,0.24091
+precision,0.13789,0.10187,0.08279,0.05614,0.03539,0.01756,0.0098,0.00526,0.00116
+recall,0.12029,0.26318,0.35262,0.47301,0.59311,0.73334,0.81301,0.87032,0.95131
+mrr,0.13789,0.20471,0.22654,0.24257,0.25099,0.25549,0.25655,0.25695,0.25715
+recall_cap,0.13789,0.26328,0.35262,0.47301,0.59311,0.73334,0.81301,0.87032,0.95131
+hole,0.85863,0.89484,0.91385,0.94056,0.9612,0.97919,0.98709,0.99174,0.99609

beir_output/nq.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "nq",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.13789,
+            "NDCG@3": 0.20975,
+            "NDCG@5": 0.24822,
+            "NDCG@10": 0.28918,
+            "NDCG@20": 0.32125,
+            "NDCG@50": 0.3507,
+            "NDCG@100": 0.36448,
+            "NDCG@200": 0.37301,
+            "NDCG@1000": 0.38303
+        },
+        "map": {
+            "MAP@1": 0.12029,
+            "MAP@3": 0.18428,
+            "MAP@5": 0.20611,
+            "MAP@10": 0.22412,
+            "MAP@20": 0.23357,
+            "MAP@50": 0.23878,
+            "MAP@100": 0.24015,
+            "MAP@200": 0.24064,
+            "MAP@1000": 0.24091
+        },
+        "precision": {
+            "P@1": 0.13789,
+            "P@3": 0.10187,
+            "P@5": 0.08279,
+            "P@10": 0.05614,
+            "P@20": 0.03539,
+            "P@50": 0.01756,
+            "P@100": 0.0098,
+            "P@200": 0.00526,
+            "P@1000": 0.00116
+        },
+        "recall": {
+            "Recall@1": 0.12029,
+            "Recall@3": 0.26318,
+            "Recall@5": 0.35262,
+            "Recall@10": 0.47301,
+            "Recall@20": 0.59311,
+            "Recall@50": 0.73334,
+            "Recall@100": 0.81301,
+            "Recall@200": 0.87032,
+            "Recall@1000": 0.95131
+        },
+        "mrr": {
+            "MRR@1": 0.13789,
+            "MRR@3": 0.20471,
+            "MRR@5": 0.22654,
+            "MRR@10": 0.24257,
+            "MRR@20": 0.25099,
+            "MRR@50": 0.25549,
+            "MRR@100": 0.25655,
+            "MRR@200": 0.25695,
+            "MRR@1000": 0.25715
+        },
+        "recall_cap": {
+            "R_cap@1": 0.13789,
+            "R_cap@3": 0.26328,
+            "R_cap@5": 0.35262,
+            "R_cap@10": 0.47301,
+            "R_cap@20": 0.59311,
+            "R_cap@50": 0.73334,
+            "R_cap@100": 0.81301,
+            "R_cap@200": 0.87032,
+            "R_cap@1000": 0.95131
+        },
+        "hole": {
+            "Hole@1": 0.85863,
+            "Hole@3": 0.89484,
+            "Hole@5": 0.91385,
+            "Hole@10": 0.94056,
+            "Hole@20": 0.9612,
+            "Hole@50": 0.97919,
+            "Hole@100": 0.98709,
+            "Hole@200": 0.99174,
+            "Hole@1000": 0.99609
+        }
+    }
+}

beir_output/quora.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.6863,0.74425,0.7668,0.78744,0.799,0.80727,0.80986,0.81162,0.8131
+map,0.59778,0.70299,0.72323,0.73668,0.74201,0.74452,0.74503,0.74523,0.74533
+precision,0.6863,0.32663,0.21894,0.12262,0.0665,0.02861,0.01476,0.00756,0.00156
+recall,0.59778,0.77735,0.83657,0.89623,0.93488,0.9675,0.97907,0.98836,0.99717
+mrr,0.6859,0.75415,0.76519,0.77154,0.77347,0.77418,0.77427,0.77431,0.77433
+recall_cap,0.6859,0.79025,0.84164,0.89764,0.93518,0.96753,0.97907,0.98836,0.99717
+hole,0.2928,0.63867,0.74272,0.83542,0.89072,0.92872,0.94389,0.95314,0.96223

beir_output/quora.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "quora",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.6863,
+            "NDCG@3": 0.74425,
+            "NDCG@5": 0.7668,
+            "NDCG@10": 0.78744,
+            "NDCG@20": 0.799,
+            "NDCG@50": 0.80727,
+            "NDCG@100": 0.80986,
+            "NDCG@200": 0.81162,
+            "NDCG@1000": 0.8131
+        },
+        "map": {
+            "MAP@1": 0.59778,
+            "MAP@3": 0.70299,
+            "MAP@5": 0.72323,
+            "MAP@10": 0.73668,
+            "MAP@20": 0.74201,
+            "MAP@50": 0.74452,
+            "MAP@100": 0.74503,
+            "MAP@200": 0.74523,
+            "MAP@1000": 0.74533
+        },
+        "precision": {
+            "P@1": 0.6863,
+            "P@3": 0.32663,
+            "P@5": 0.21894,
+            "P@10": 0.12262,
+            "P@20": 0.0665,
+            "P@50": 0.02861,
+            "P@100": 0.01476,
+            "P@200": 0.00756,
+            "P@1000": 0.00156
+        },
+        "recall": {
+            "Recall@1": 0.59778,
+            "Recall@3": 0.77735,
+            "Recall@5": 0.83657,
+            "Recall@10": 0.89623,
+            "Recall@20": 0.93488,
+            "Recall@50": 0.9675,
+            "Recall@100": 0.97907,
+            "Recall@200": 0.98836,
+            "Recall@1000": 0.99717
+        },
+        "mrr": {
+            "MRR@1": 0.6859,
+            "MRR@3": 0.75415,
+            "MRR@5": 0.76519,
+            "MRR@10": 0.77154,
+            "MRR@20": 0.77347,
+            "MRR@50": 0.77418,
+            "MRR@100": 0.77427,
+            "MRR@200": 0.77431,
+            "MRR@1000": 0.77433
+        },
+        "recall_cap": {
+            "R_cap@1": 0.6859,
+            "R_cap@3": 0.79025,
+            "R_cap@5": 0.84164,
+            "R_cap@10": 0.89764,
+            "R_cap@20": 0.93518,
+            "R_cap@50": 0.96753,
+            "R_cap@100": 0.97907,
+            "R_cap@200": 0.98836,
+            "R_cap@1000": 0.99717
+        },
+        "hole": {
+            "Hole@1": 0.2928,
+            "Hole@3": 0.63867,
+            "Hole@5": 0.74272,
+            "Hole@10": 0.83542,
+            "Hole@20": 0.89072,
+            "Hole@50": 0.92872,
+            "Hole@100": 0.94389,
+            "Hole@200": 0.95314,
+            "Hole@1000": 0.96223
+        }
+    }
+}

beir_output/scidocs.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.181,0.14224,0.12576,0.15287,0.17834,0.20556,0.22534,0.24141,0.2773
+map,0.03683,0.06302,0.07547,0.08755,0.09599,0.1019,0.10468,0.10598,0.10736
+precision,0.181,0.131,0.1098,0.0795,0.0544,0.0298,0.01846,0.0109,0.0031
+recall,0.03683,0.07998,0.11153,0.16118,0.22043,0.30205,0.37422,0.4427,0.6287
+mrr,0.181,0.239,0.25555,0.26985,0.2777,0.28135,0.2823,0.28278,0.28309
+recall_cap,0.181,0.131,0.11153,0.16118,0.22043,0.30205,0.37422,0.4427,0.6287
+hole,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0

beir_output/scidocs.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "scidocs",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.181,
+            "NDCG@3": 0.14224,
+            "NDCG@5": 0.12576,
+            "NDCG@10": 0.15287,
+            "NDCG@20": 0.17834,
+            "NDCG@50": 0.20556,
+            "NDCG@100": 0.22534,
+            "NDCG@200": 0.24141,
+            "NDCG@1000": 0.2773
+        },
+        "map": {
+            "MAP@1": 0.03683,
+            "MAP@3": 0.06302,
+            "MAP@5": 0.07547,
+            "MAP@10": 0.08755,
+            "MAP@20": 0.09599,
+            "MAP@50": 0.1019,
+            "MAP@100": 0.10468,
+            "MAP@200": 0.10598,
+            "MAP@1000": 0.10736
+        },
+        "precision": {
+            "P@1": 0.181,
+            "P@3": 0.131,
+            "P@5": 0.1098,
+            "P@10": 0.0795,
+            "P@20": 0.0544,
+            "P@50": 0.0298,
+            "P@100": 0.01846,
+            "P@200": 0.0109,
+            "P@1000": 0.0031
+        },
+        "recall": {
+            "Recall@1": 0.03683,
+            "Recall@3": 0.07998,
+            "Recall@5": 0.11153,
+            "Recall@10": 0.16118,
+            "Recall@20": 0.22043,
+            "Recall@50": 0.30205,
+            "Recall@100": 0.37422,
+            "Recall@200": 0.4427,
+            "Recall@1000": 0.6287
+        },
+        "mrr": {
+            "MRR@1": 0.181,
+            "MRR@3": 0.239,
+            "MRR@5": 0.25555,
+            "MRR@10": 0.26985,
+            "MRR@20": 0.2777,
+            "MRR@50": 0.28135,
+            "MRR@100": 0.2823,
+            "MRR@200": 0.28278,
+            "MRR@1000": 0.28309
+        },
+        "recall_cap": {
+            "R_cap@1": 0.181,
+            "R_cap@3": 0.131,
+            "R_cap@5": 0.11153,
+            "R_cap@10": 0.16118,
+            "R_cap@20": 0.22043,
+            "R_cap@50": 0.30205,
+            "R_cap@100": 0.37422,
+            "R_cap@200": 0.4427,
+            "R_cap@1000": 0.6287
+        },
+        "hole": {
+            "Hole@1": 0.0,
+            "Hole@3": 0.0,
+            "Hole@5": 0.0,
+            "Hole@10": 0.0,
+            "Hole@20": 0.0,
+            "Hole@50": 0.0,
+            "Hole@100": 0.0,
+            "Hole@200": 0.0,
+            "Hole@1000": 0.0
+        }
+    }
+}

beir_output/scifact.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.51333,0.59736,0.63108,0.65164,0.6611,0.67646,0.68093,0.68372,0.6888
+map,0.49289,0.56855,0.59006,0.60046,0.6032,0.60601,0.60644,0.6066,0.60672
+precision,0.51333,0.23667,0.16267,0.089,0.04667,0.02047,0.01053,0.00538,0.00112
+recall,0.49289,0.65267,0.73639,0.79467,0.83133,0.90489,0.93156,0.95056,0.99333
+mrr,0.51333,0.58667,0.60483,0.6124,0.61458,0.61678,0.61713,0.61728,0.61739
+recall_cap,0.51333,0.65611,0.73639,0.79467,0.83133,0.90489,0.93156,0.95056,0.99333
+hole,0.46667,0.72889,0.79467,0.859,0.891,0.91687,0.9281,0.93337,0.94051

beir_output/scifact.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "scifact",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.51333,
+            "NDCG@3": 0.59736,
+            "NDCG@5": 0.63108,
+            "NDCG@10": 0.65164,
+            "NDCG@20": 0.6611,
+            "NDCG@50": 0.67646,
+            "NDCG@100": 0.68093,
+            "NDCG@200": 0.68372,
+            "NDCG@1000": 0.6888
+        },
+        "map": {
+            "MAP@1": 0.49289,
+            "MAP@3": 0.56855,
+            "MAP@5": 0.59006,
+            "MAP@10": 0.60046,
+            "MAP@20": 0.6032,
+            "MAP@50": 0.60601,
+            "MAP@100": 0.60644,
+            "MAP@200": 0.6066,
+            "MAP@1000": 0.60672
+        },
+        "precision": {
+            "P@1": 0.51333,
+            "P@3": 0.23667,
+            "P@5": 0.16267,
+            "P@10": 0.089,
+            "P@20": 0.04667,
+            "P@50": 0.02047,
+            "P@100": 0.01053,
+            "P@200": 0.00538,
+            "P@1000": 0.00112
+        },
+        "recall": {
+            "Recall@1": 0.49289,
+            "Recall@3": 0.65267,
+            "Recall@5": 0.73639,
+            "Recall@10": 0.79467,
+            "Recall@20": 0.83133,
+            "Recall@50": 0.90489,
+            "Recall@100": 0.93156,
+            "Recall@200": 0.95056,
+            "Recall@1000": 0.99333
+        },
+        "mrr": {
+            "MRR@1": 0.51333,
+            "MRR@3": 0.58667,
+            "MRR@5": 0.60483,
+            "MRR@10": 0.6124,
+            "MRR@20": 0.61458,
+            "MRR@50": 0.61678,
+            "MRR@100": 0.61713,
+            "MRR@200": 0.61728,
+            "MRR@1000": 0.61739
+        },
+        "recall_cap": {
+            "R_cap@1": 0.51333,
+            "R_cap@3": 0.65611,
+            "R_cap@5": 0.73639,
+            "R_cap@10": 0.79467,
+            "R_cap@20": 0.83133,
+            "R_cap@50": 0.90489,
+            "R_cap@100": 0.93156,
+            "R_cap@200": 0.95056,
+            "R_cap@1000": 0.99333
+        },
+        "hole": {
+            "Hole@1": 0.46667,
+            "Hole@3": 0.72889,
+            "Hole@5": 0.79467,
+            "Hole@10": 0.859,
+            "Hole@20": 0.891,
+            "Hole@50": 0.91687,
+            "Hole@100": 0.9281,
+            "Hole@200": 0.93337,
+            "Hole@1000": 0.94051
+        }
+    }
+}

beir_output/senteval-core.json

@@ -0,0 +1,5 @@
+{
+    "eval_senteval-stsb_spearman": 0.7786676025109732,
+    "eval_senteval-sickr_spearman": 0.7166329672415084,
+    "eval_senteval-avg_sts": 0.7476502848762407
+}

beir_output/senteval.json

@@ -0,0 +1,99 @@
+{
+    "STSBenchmark": {
+        "train": {
+            "pearson": [
+                0.7593357300138104,
+                0.0
+            ],
+            "spearman": [
+                0.7363780516874698,
+                0.0
+            ],
+            "nsamples": 5749
+        },
+        "dev": {
+            "pearson": [
+                0.7708717320377951,
+                1.0423163280003215e-295
+            ],
+            "spearman": [
+                0.7786676025109732,
+                1.5214906511814883e-305
+            ],
+            "nsamples": 1500
+        },
+        "test": {
+            "pearson": [
+                0.7501376173451568,
+                1.3467040400624944e-249
+            ],
+            "spearman": [
+                0.750824400583584,
+                2.649603496637417e-250
+            ],
+            "nsamples": 1379
+        },
+        "all": {
+            "pearson": {
+                "all": 0.7604793924665235,
+                "mean": 0.7601150264655875,
+                "wmean": 0.759871173415051
+            },
+            "spearman": {
+                "all": 0.7504771603185239,
+                "mean": 0.7552900182606758,
+                "wmean": 0.7460391366854991
+            }
+        }
+    },
+    "SICKRelatedness": {
+        "train": {
+            "pearson": [
+                0.7662304008155296,
+                0.0
+            ],
+            "spearman": [
+                0.6902538404584456,
+                0.0
+            ],
+            "nsamples": 4500
+        },
+        "dev": {
+            "pearson": [
+                0.7821571234476052,
+                2.2004576953729478e-104
+            ],
+            "spearman": [
+                0.7166329672415084,
+                5.843294318811089e-80
+            ],
+            "nsamples": 500
+        },
+        "test": {
+            "pearson": [
+                0.7611310123704713,
+                0.0
+            ],
+            "spearman": [
+                0.6832295457395683,
+                0.0
+            ],
+            "nsamples": 4927
+        },
+        "all": {
+            "pearson": {
+                "all": 0.7644308360476054,
+                "mean": 0.7698395122112021,
+                "wmean": 0.7645016483673817
+            },
+            "spearman": {
+                "all": 0.6882116860582081,
+                "mean": 0.6967054511465074,
+                "wmean": 0.6880961758378777
+            }
+        }
+    },
+    "eval_senteval-stsb_spearman": 0.7786676025109732,
+    "eval_senteval-sickr_spearman": 0.7166329672415084,
+    "eval_senteval-avg_sts": 0.7476502848762407
+}

beir_output/trec-covid.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.69,0.65285,0.63089,0.58524,0.5586,0.50163,0.4472,0.38654,0.41317
+map,0.00206,0.00574,0.00878,0.01465,0.02531,0.04934,0.07763,0.11328,0.18931
+precision,0.74,0.7,0.676,0.62,0.6,0.5316,0.4676,0.3855,0.18724
+recall,0.00206,0.0059,0.00939,0.01652,0.03073,0.06546,0.11007,0.17512,0.39341
+mrr,0.74,0.77333,0.79133,0.80324,0.80324,0.80376,0.80376,0.80376,0.80376
+recall_cap,0.74,0.69333,0.676,0.62,0.6,0.5316,0.4676,0.38913,0.3945
+hole,0.06,0.07333,0.08,0.11,0.133,0.166,0.208,0.2588,0.38622

beir_output/trec-covid.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "trec-covid",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.69,
+            "NDCG@3": 0.65285,
+            "NDCG@5": 0.63089,
+            "NDCG@10": 0.58524,
+            "NDCG@20": 0.5586,
+            "NDCG@50": 0.50163,
+            "NDCG@100": 0.4472,
+            "NDCG@200": 0.38654,
+            "NDCG@1000": 0.41317
+        },
+        "map": {
+            "MAP@1": 0.00206,
+            "MAP@3": 0.00574,
+            "MAP@5": 0.00878,
+            "MAP@10": 0.01465,
+            "MAP@20": 0.02531,
+            "MAP@50": 0.04934,
+            "MAP@100": 0.07763,
+            "MAP@200": 0.11328,
+            "MAP@1000": 0.18931
+        },
+        "precision": {
+            "P@1": 0.74,
+            "P@3": 0.7,
+            "P@5": 0.676,
+            "P@10": 0.62,
+            "P@20": 0.6,
+            "P@50": 0.5316,
+            "P@100": 0.4676,
+            "P@200": 0.3855,
+            "P@1000": 0.18724
+        },
+        "recall": {
+            "Recall@1": 0.00206,
+            "Recall@3": 0.0059,
+            "Recall@5": 0.00939,
+            "Recall@10": 0.01652,
+            "Recall@20": 0.03073,
+            "Recall@50": 0.06546,
+            "Recall@100": 0.11007,
+            "Recall@200": 0.17512,
+            "Recall@1000": 0.39341
+        },
+        "mrr": {
+            "MRR@1": 0.74,
+            "MRR@3": 0.77333,
+            "MRR@5": 0.79133,
+            "MRR@10": 0.80324,
+            "MRR@20": 0.80324,
+            "MRR@50": 0.80376,
+            "MRR@100": 0.80376,
+            "MRR@200": 0.80376,
+            "MRR@1000": 0.80376
+        },
+        "recall_cap": {
+            "R_cap@1": 0.74,
+            "R_cap@3": 0.69333,
+            "R_cap@5": 0.676,
+            "R_cap@10": 0.62,
+            "R_cap@20": 0.6,
+            "R_cap@50": 0.5316,
+            "R_cap@100": 0.4676,
+            "R_cap@200": 0.38913,
+            "R_cap@1000": 0.3945
+        },
+        "hole": {
+            "Hole@1": 0.06,
+            "Hole@3": 0.07333,
+            "Hole@5": 0.08,
+            "Hole@10": 0.11,
+            "Hole@20": 0.133,
+            "Hole@50": 0.166,
+            "Hole@100": 0.208,
+            "Hole@200": 0.2588,
+            "Hole@1000": 0.38622
+        }
+    }
+}

beir_output/webis-touche2020.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.20408,0.18523,0.1866,0.19644,0.20838,0.26475,0.31201,0.35073,0.43441
+map,0.01734,0.03062,0.04306,0.07298,0.09244,0.11281,0.1244,0.1308,0.13959
+precision,0.20408,0.19048,0.18776,0.17959,0.1398,0.0951,0.06735,0.04357,0.01473
+recall,0.01734,0.04371,0.07424,0.13637,0.20649,0.31943,0.43738,0.53544,0.80059
+mrr,0.20408,0.32993,0.37177,0.37659,0.3843,0.38601,0.38646,0.38671,0.38671
+recall_cap,0.20408,0.19048,0.18776,0.19683,0.21249,0.31943,0.43738,0.53544,0.80059
+hole,0.73469,0.7483,0.75102,0.77551,0.82245,0.87184,0.90347,0.93133,0.96878

beir_output/webis-touche2020.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "webis-touche2020",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.20408,
+            "NDCG@3": 0.18523,
+            "NDCG@5": 0.1866,
+            "NDCG@10": 0.19644,
+            "NDCG@20": 0.20838,
+            "NDCG@50": 0.26475,
+            "NDCG@100": 0.31201,
+            "NDCG@200": 0.35073,
+            "NDCG@1000": 0.43441
+        },
+        "map": {
+            "MAP@1": 0.01734,
+            "MAP@3": 0.03062,
+            "MAP@5": 0.04306,
+            "MAP@10": 0.07298,
+            "MAP@20": 0.09244,
+            "MAP@50": 0.11281,
+            "MAP@100": 0.1244,
+            "MAP@200": 0.1308,
+            "MAP@1000": 0.13959
+        },
+        "precision": {
+            "P@1": 0.20408,
+            "P@3": 0.19048,
+            "P@5": 0.18776,
+            "P@10": 0.17959,
+            "P@20": 0.1398,
+            "P@50": 0.0951,
+            "P@100": 0.06735,
+            "P@200": 0.04357,
+            "P@1000": 0.01473
+        },
+        "recall": {
+            "Recall@1": 0.01734,
+            "Recall@3": 0.04371,
+            "Recall@5": 0.07424,
+            "Recall@10": 0.13637,
+            "Recall@20": 0.20649,
+            "Recall@50": 0.31943,
+            "Recall@100": 0.43738,
+            "Recall@200": 0.53544,
+            "Recall@1000": 0.80059
+        },
+        "mrr": {
+            "MRR@1": 0.20408,
+            "MRR@3": 0.32993,
+            "MRR@5": 0.37177,
+            "MRR@10": 0.37659,
+            "MRR@20": 0.3843,
+            "MRR@50": 0.38601,
+            "MRR@100": 0.38646,
+            "MRR@200": 0.38671,
+            "MRR@1000": 0.38671
+        },
+        "recall_cap": {
+            "R_cap@1": 0.20408,
+            "R_cap@3": 0.19048,
+            "R_cap@5": 0.18776,
+            "R_cap@10": 0.19683,
+            "R_cap@20": 0.21249,
+            "R_cap@50": 0.31943,
+            "R_cap@100": 0.43738,
+            "R_cap@200": 0.53544,
+            "R_cap@1000": 0.80059
+        },
+        "hole": {
+            "Hole@1": 0.73469,
+            "Hole@3": 0.7483,
+            "Hole@5": 0.75102,
+            "Hole@10": 0.77551,
+            "Hole@20": 0.82245,
+            "Hole@50": 0.87184,
+            "Hole@100": 0.90347,
+            "Hole@200": 0.93133,
+            "Hole@1000": 0.96878
+        }
+    }
+}

cc.moco.RC20+Qext+title+T0gen.bs1024.gpu8.sh

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+
+export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
+export LOCAL_RANK=0
+export WORLD_SIZE=8
+
+export TOKENIZERS_PARALLELISM=false
+export NUM_WORKER=8
+export MAX_STEPS=100000
+
+export CUDA_LAUNCH_BLOCKING=1
+export NCCL_DEBUG=INFO
+
+export EXP_NAME=cc-hybrid.RC20+Qext10+title16+T0gen64.seed477.moco-2e14.contriever256-special50-titles_special.bert-base-uncased.avg.dot.q128d256.step100k.bs1024.lr5e5
+export PROJECT_DIR=output_dir/$EXP_NAME
+mkdir -p $PROJECT_DIR
+cp "$0" $PROJECT_DIR  # copy bash to project dir
+echo $PROJECT_DIR
+
+export WANDB_NAME=$EXP_NAME
+export WANDB_PROJECT=unsup_retrieval_cc
+export WANDB_DIR=$PROJECT_DIR
+mkdir -p $WANDB_DIR/wandb
+
+
+nohup python -m torch.distributed.launch --nproc_per_node=8 --master_port=3112 --max_restarts=0 train.py --model_name_or_path bert-base-uncased --arch_type moco --q_extract google/t5-small-lm-adapt --num_random_chunk 16 --q_extract_ratio 0.1 --train_file /export/home/data/search/upr/cc/T03B_PileCC_title.json:/export/home/data/search/upr/cc/T03B_PileCC_topic.json:/export/home/data/search/upr/cc/T03B_PileCC_exsum.json --dev_file /export/home/data/pretrain/wiki2021_structure/wiki_psgs_w100.dev.tail2e13.tsv --data_type hf --data_pipeline_name contriever256-special80%-titles_special --remove_unused_columns False --sim_type dot --queue_size 16384 --momentum 0.9995 --output_dir $PROJECT_DIR --overwrite_output_dir --cache_dir /export/home/data/pretrain/.cache --max_steps $MAX_STEPS --warmup_steps 10000 --logging_steps 100 --eval_steps 10000 --save_steps 100000000 --per_device_train_batch_size 128 --per_device_eval_batch_size 128 --beir_batch_size 256 --dataloader_num_workers $NUM_WORKER --learning_rate 5e-5 --max_q_tokens 128 --max_d_tokens 256 --evaluation_strategy steps --load_best_model_at_end --do_train --do_eval --run_name $EXP_NAME --fp16 --seed 477 --report_to wandb --wiki_passage_path /export/home/data/search/nq/psgs_w100.tsv --qa_datasets_path /export/home/data/search/nq/qas/*-test.csv,/export/home/data/search/nq/qas/entityqs/test/P*.test.json > $PROJECT_DIR/nohup.log 2>&1 & echo $! > run.pid
+
+#export WANDB_RUN_ID=1itmx6a9
+#nohup python -m torch.distributed.launch --nproc_per_node=8 --master_port=3112 --max_restarts=0 train.py --model_name_or_path bert-base-uncased --arch_type inbatch --reload_model_from $PROJECT_DIR --remove_unused_columns False --sim_type dot --output_dir $PROJECT_DIR --cache_dir /export/home/data/pretrain/.cache --max_steps $MAX_STEPS --do_eval --run_name $EXP_NAME --fp16 --seed 477 --report_to wandb --wiki_passage_path /export/home/data/search/nq/psgs_w100.tsv --qa_datasets_path /export/home/data/search/nq/qas/*-test.csv,/export/home/data/search/nq/qas/entityqs/test/P*.test.json > $PROJECT_DIR/nohup-eval.log 2>&1 & echo $! > run.pid
+

eval_output/checkpoint-100000/arguana.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.16501,0.27175,0.31815,0.37955,0.4119,0.43478,0.44184,0.44581,0.44949
+map,0.16501,0.24526,0.2709,0.29624,0.30539,0.30919,0.30982,0.31002,0.31011
+precision,0.16501,0.11617,0.09232,0.06515,0.03887,0.01782,0.00935,0.00482,0.00099
+recall,0.16501,0.34851,0.46159,0.65149,0.77738,0.89118,0.93457,0.96302,0.99431
+mrr,0.1707,0.24798,0.27291,0.29858,0.30765,0.31146,0.31208,0.31228,0.31237
+recall_cap,0.1707,0.34993,0.46017,0.6522,0.77738,0.89189,0.93457,0.96302,0.99431
+hole,0.67568,0.68753,0.68834,0.67817,0.7181,0.79438,0.83069,0.85393,0.87081

eval_output/checkpoint-100000/arguana.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "arguana",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.16501,
+            "NDCG@3": 0.27175,
+            "NDCG@5": 0.31815,
+            "NDCG@10": 0.37955,
+            "NDCG@20": 0.4119,
+            "NDCG@50": 0.43478,
+            "NDCG@100": 0.44184,
+            "NDCG@200": 0.44581,
+            "NDCG@1000": 0.44949
+        },
+        "map": {
+            "MAP@1": 0.16501,
+            "MAP@3": 0.24526,
+            "MAP@5": 0.2709,
+            "MAP@10": 0.29624,
+            "MAP@20": 0.30539,
+            "MAP@50": 0.30919,
+            "MAP@100": 0.30982,
+            "MAP@200": 0.31002,
+            "MAP@1000": 0.31011
+        },
+        "precision": {
+            "P@1": 0.16501,
+            "P@3": 0.11617,
+            "P@5": 0.09232,
+            "P@10": 0.06515,
+            "P@20": 0.03887,
+            "P@50": 0.01782,
+            "P@100": 0.00935,
+            "P@200": 0.00482,
+            "P@1000": 0.00099
+        },
+        "recall": {
+            "Recall@1": 0.16501,
+            "Recall@3": 0.34851,
+            "Recall@5": 0.46159,
+            "Recall@10": 0.65149,
+            "Recall@20": 0.77738,
+            "Recall@50": 0.89118,
+            "Recall@100": 0.93457,
+            "Recall@200": 0.96302,
+            "Recall@1000": 0.99431
+        },
+        "mrr": {
+            "MRR@1": 0.1707,
+            "MRR@3": 0.24798,
+            "MRR@5": 0.27291,
+            "MRR@10": 0.29858,
+            "MRR@20": 0.30765,
+            "MRR@50": 0.31146,
+            "MRR@100": 0.31208,
+            "MRR@200": 0.31228,
+            "MRR@1000": 0.31237
+        },
+        "recall_cap": {
+            "R_cap@1": 0.1707,
+            "R_cap@3": 0.34993,
+            "R_cap@5": 0.46017,
+            "R_cap@10": 0.6522,
+            "R_cap@20": 0.77738,
+            "R_cap@50": 0.89189,
+            "R_cap@100": 0.93457,
+            "R_cap@200": 0.96302,
+            "R_cap@1000": 0.99431
+        },
+        "hole": {
+            "Hole@1": 0.67568,
+            "Hole@3": 0.68753,
+            "Hole@5": 0.68834,
+            "Hole@10": 0.67817,
+            "Hole@20": 0.7181,
+            "Hole@50": 0.79438,
+            "Hole@100": 0.83069,
+            "Hole@200": 0.85393,
+            "Hole@1000": 0.87081
+        }
+    }
+}

eval_output/checkpoint-100000/beir.json

@@ -0,0 +1,46 @@
+{
+    "eval_beir-nfcorpus_ndcg@10": 0.30262,
+    "eval_beir-nfcorpus_recall@10": 0.15211,
+    "eval_beir-nfcorpus_recall@20": 0.18187,
+    "eval_beir-nfcorpus_recall@100": 0.28685,
+    "eval_beir-fiqa_ndcg@10": 0.25245,
+    "eval_beir-fiqa_recall@10": 0.31073,
+    "eval_beir-fiqa_recall@20": 0.37975,
+    "eval_beir-fiqa_recall@100": 0.57848,
+    "eval_beir-arguana_ndcg@10": 0.37955,
+    "eval_beir-arguana_recall@10": 0.65149,
+    "eval_beir-arguana_recall@20": 0.77738,
+    "eval_beir-arguana_recall@100": 0.93457,
+    "eval_beir-scidocs_ndcg@10": 0.15287,
+    "eval_beir-scidocs_recall@10": 0.16118,
+    "eval_beir-scidocs_recall@20": 0.22043,
+    "eval_beir-scidocs_recall@100": 0.37422,
+    "eval_beir-scifact_ndcg@10": 0.65164,
+    "eval_beir-scifact_recall@10": 0.79467,
+    "eval_beir-scifact_recall@20": 0.83133,
+    "eval_beir-scifact_recall@100": 0.93156,
+    "eval_beir-webis-touche2020_ndcg@10": 0.19644,
+    "eval_beir-webis-touche2020_recall@10": 0.13637,
+    "eval_beir-webis-touche2020_recall@20": 0.20649,
+    "eval_beir-webis-touche2020_recall@100": 0.43738,
+    "eval_beir-cqadupstack_ndcg@10": 0.264705,
+    "eval_beir-cqadupstack_recall@10": 0.3632225,
+    "eval_beir-cqadupstack_recall@20": 0.4335508333333334,
+    "eval_beir-cqadupstack_recall@100": 0.6014158333333334,
+    "eval_beir-trec-covid_ndcg@10": 0.58524,
+    "eval_beir-trec-covid_recall@10": 0.62,
+    "eval_beir-trec-covid_recall@20": 0.6,
+    "eval_beir-trec-covid_recall@100": 0.4676,
+    "eval_beir-quora_ndcg@10": 0.78744,
+    "eval_beir-quora_recall@10": 0.89623,
+    "eval_beir-quora_recall@20": 0.93488,
+    "eval_beir-quora_recall@100": 0.97907,
+    "eval_beir-nq_ndcg@10": 0.28918,
+    "eval_beir-nq_recall@10": 0.47301,
+    "eval_beir-nq_recall@20": 0.59311,
+    "eval_beir-nq_recall@100": 0.81301,
+    "eval_beir-avg_ndcg@10": 0.3862135,
+    "eval_beir-avg_recall@10": 0.45590125000000004,
+    "eval_beir-avg_recall@20": 0.5158790833333333,
+    "eval_beir-avg_recall@100": 0.6404155833333334
+}

eval_output/checkpoint-100000/cqadupstack.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.18406,0.2229525,0.24138916666666663,0.264705,0.2836966666666667,0.3036575,0.317455,0.3279316666666667,0.3471208333333333
+map,0.15620583333333335,0.19959000000000002,0.21138416666666668,0.22192916666666665,0.22766583333333335,0.23130833333333337,0.2327866666666667,0.2334733333333333,0.23405083333333332
+precision,0.18406,0.10411250000000001,0.0760425,0.04850833333333333,0.0298675,0.015099166666666665,0.008941666666666665,0.005136666666666667,0.0013391666666666667
+recall,0.15620583333333335,0.24788249999999998,0.2942608333333333,0.3632225,0.4335508333333334,0.5252308333333333,0.6014158333333334,0.6680558333333332,0.8154633333333333
+mrr,0.18406,0.23019749999999997,0.24204666666666663,0.25234,0.257585,0.2605608333333334,0.26161333333333336,0.26206250000000003,0.26239833333333334
+recall_cap,0.18406,0.2531541666666666,0.2966533333333334,0.3642716666666666,0.43398416666666667,0.5253808333333334,0.6014775,0.6680633333333333,0.8154633333333333
+hole,0.7429125,0.8188533333333333,0.8458958333333334,0.8714099999999999,0.8905433333333334,0.9064025,0.9140950000000001,0.9195925000000003,0.927175

eval_output/checkpoint-100000/cqadupstack.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "cqadupstack",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.18406,
+            "NDCG@3": 0.2229525,
+            "NDCG@5": 0.24138916666666663,
+            "NDCG@10": 0.264705,
+            "NDCG@20": 0.2836966666666667,
+            "NDCG@50": 0.3036575,
+            "NDCG@100": 0.317455,
+            "NDCG@200": 0.3279316666666667,
+            "NDCG@1000": 0.3471208333333333
+        },
+        "map": {
+            "MAP@1": 0.15620583333333335,
+            "MAP@3": 0.19959000000000002,
+            "MAP@5": 0.21138416666666668,
+            "MAP@10": 0.22192916666666665,
+            "MAP@20": 0.22766583333333335,
+            "MAP@50": 0.23130833333333337,
+            "MAP@100": 0.2327866666666667,
+            "MAP@200": 0.2334733333333333,
+            "MAP@1000": 0.23405083333333332
+        },
+        "precision": {
+            "P@1": 0.18406,
+            "P@3": 0.10411250000000001,
+            "P@5": 0.0760425,
+            "P@10": 0.04850833333333333,
+            "P@20": 0.0298675,
+            "P@50": 0.015099166666666665,
+            "P@100": 0.008941666666666665,
+            "P@200": 0.005136666666666667,
+            "P@1000": 0.0013391666666666667
+        },
+        "recall": {
+            "Recall@1": 0.15620583333333335,
+            "Recall@3": 0.24788249999999998,
+            "Recall@5": 0.2942608333333333,
+            "Recall@10": 0.3632225,
+            "Recall@20": 0.4335508333333334,
+            "Recall@50": 0.5252308333333333,
+            "Recall@100": 0.6014158333333334,
+            "Recall@200": 0.6680558333333332,
+            "Recall@1000": 0.8154633333333333
+        },
+        "mrr": {
+            "MRR@1": 0.18406,
+            "MRR@3": 0.23019749999999997,
+            "MRR@5": 0.24204666666666663,
+            "MRR@10": 0.25234,
+            "MRR@20": 0.257585,
+            "MRR@50": 0.2605608333333334,
+            "MRR@100": 0.26161333333333336,
+            "MRR@200": 0.26206250000000003,
+            "MRR@1000": 0.26239833333333334
+        },
+        "recall_cap": {
+            "R_cap@1": 0.18406,
+            "R_cap@3": 0.2531541666666666,
+            "R_cap@5": 0.2966533333333334,
+            "R_cap@10": 0.3642716666666666,
+            "R_cap@20": 0.43398416666666667,
+            "R_cap@50": 0.5253808333333334,
+            "R_cap@100": 0.6014775,
+            "R_cap@200": 0.6680633333333333,
+            "R_cap@1000": 0.8154633333333333
+        },
+        "hole": {
+            "Hole@1": 0.7429125,
+            "Hole@3": 0.8188533333333333,
+            "Hole@5": 0.8458958333333334,
+            "Hole@10": 0.8714099999999999,
+            "Hole@20": 0.8905433333333334,
+            "Hole@50": 0.9064025,
+            "Hole@100": 0.9140950000000001,
+            "Hole@200": 0.9195925000000003,
+            "Hole@1000": 0.927175
+        }
+    }
+}

eval_output/checkpoint-100000/fiqa.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.23302,0.2207,0.22989,0.25245,0.27483,0.30454,0.32168,0.33558,0.36481
+map,0.1186,0.16754,0.18035,0.19302,0.20042,0.20645,0.20851,0.2096,0.21068
+precision,0.23302,0.1466,0.10772,0.07068,0.04414,0.02352,0.01384,0.00802,0.00216
+recall,0.1186,0.20366,0.24624,0.31073,0.37975,0.49631,0.57848,0.65152,0.84222
+mrr,0.23302,0.28627,0.29992,0.3112,0.31809,0.32196,0.32302,0.32337,0.32368
+recall_cap,0.23302,0.22659,0.25239,0.31126,0.37975,0.49631,0.57848,0.65152,0.84222
+hole,0.71296,0.79681,0.83333,0.8679,0.89367,0.91238,0.92168,0.92719,0.93598

eval_output/checkpoint-100000/fiqa.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "fiqa",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.23302,
+            "NDCG@3": 0.2207,
+            "NDCG@5": 0.22989,
+            "NDCG@10": 0.25245,
+            "NDCG@20": 0.27483,
+            "NDCG@50": 0.30454,
+            "NDCG@100": 0.32168,
+            "NDCG@200": 0.33558,
+            "NDCG@1000": 0.36481
+        },
+        "map": {
+            "MAP@1": 0.1186,
+            "MAP@3": 0.16754,
+            "MAP@5": 0.18035,
+            "MAP@10": 0.19302,
+            "MAP@20": 0.20042,
+            "MAP@50": 0.20645,
+            "MAP@100": 0.20851,
+            "MAP@200": 0.2096,
+            "MAP@1000": 0.21068
+        },
+        "precision": {
+            "P@1": 0.23302,
+            "P@3": 0.1466,
+            "P@5": 0.10772,
+            "P@10": 0.07068,
+            "P@20": 0.04414,
+            "P@50": 0.02352,
+            "P@100": 0.01384,
+            "P@200": 0.00802,
+            "P@1000": 0.00216
+        },
+        "recall": {
+            "Recall@1": 0.1186,
+            "Recall@3": 0.20366,
+            "Recall@5": 0.24624,
+            "Recall@10": 0.31073,
+            "Recall@20": 0.37975,
+            "Recall@50": 0.49631,
+            "Recall@100": 0.57848,
+            "Recall@200": 0.65152,
+            "Recall@1000": 0.84222
+        },
+        "mrr": {
+            "MRR@1": 0.23302,
+            "MRR@3": 0.28627,
+            "MRR@5": 0.29992,
+            "MRR@10": 0.3112,
+            "MRR@20": 0.31809,
+            "MRR@50": 0.32196,
+            "MRR@100": 0.32302,
+            "MRR@200": 0.32337,
+            "MRR@1000": 0.32368
+        },
+        "recall_cap": {
+            "R_cap@1": 0.23302,
+            "R_cap@3": 0.22659,
+            "R_cap@5": 0.25239,
+            "R_cap@10": 0.31126,
+            "R_cap@20": 0.37975,
+            "R_cap@50": 0.49631,
+            "R_cap@100": 0.57848,
+            "R_cap@200": 0.65152,
+            "R_cap@1000": 0.84222
+        },
+        "hole": {
+            "Hole@1": 0.71296,
+            "Hole@3": 0.79681,
+            "Hole@5": 0.83333,
+            "Hole@10": 0.8679,
+            "Hole@20": 0.89367,
+            "Hole@50": 0.91238,
+            "Hole@100": 0.92168,
+            "Hole@200": 0.92719,
+            "Hole@1000": 0.93598
+        }
+    }
+}

eval_output/checkpoint-100000/nfcorpus.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.36068,0.34345,0.33229,0.30262,0.27932,0.27176,0.27884,0.29539,0.36641
+map,0.03847,0.07726,0.09253,0.10631,0.11761,0.12837,0.13406,0.13904,0.14774
+precision,0.3808,0.33333,0.29907,0.22786,0.16594,0.10854,0.07372,0.04961,0.02026
+recall,0.03847,0.09245,0.1199,0.15211,0.18187,0.24018,0.28685,0.35578,0.60393
+mrr,0.387,0.46078,0.47843,0.48676,0.48949,0.49217,0.49262,0.49294,0.49317
+recall_cap,0.387,0.35191,0.33359,0.29074,0.26102,0.27139,0.30159,0.36072,0.60393
+hole,0.08359,0.08153,0.08607,0.10588,0.11254,0.12297,0.12409,0.12789,0.13433

eval_output/checkpoint-100000/nfcorpus.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "nfcorpus",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.36068,
+            "NDCG@3": 0.34345,
+            "NDCG@5": 0.33229,
+            "NDCG@10": 0.30262,
+            "NDCG@20": 0.27932,
+            "NDCG@50": 0.27176,
+            "NDCG@100": 0.27884,
+            "NDCG@200": 0.29539,
+            "NDCG@1000": 0.36641
+        },
+        "map": {
+            "MAP@1": 0.03847,
+            "MAP@3": 0.07726,
+            "MAP@5": 0.09253,
+            "MAP@10": 0.10631,
+            "MAP@20": 0.11761,
+            "MAP@50": 0.12837,
+            "MAP@100": 0.13406,
+            "MAP@200": 0.13904,
+            "MAP@1000": 0.14774
+        },
+        "precision": {
+            "P@1": 0.3808,
+            "P@3": 0.33333,
+            "P@5": 0.29907,
+            "P@10": 0.22786,
+            "P@20": 0.16594,
+            "P@50": 0.10854,
+            "P@100": 0.07372,
+            "P@200": 0.04961,
+            "P@1000": 0.02026
+        },
+        "recall": {
+            "Recall@1": 0.03847,
+            "Recall@3": 0.09245,
+            "Recall@5": 0.1199,
+            "Recall@10": 0.15211,
+            "Recall@20": 0.18187,
+            "Recall@50": 0.24018,
+            "Recall@100": 0.28685,
+            "Recall@200": 0.35578,
+            "Recall@1000": 0.60393
+        },
+        "mrr": {
+            "MRR@1": 0.387,
+            "MRR@3": 0.46078,
+            "MRR@5": 0.47843,
+            "MRR@10": 0.48676,
+            "MRR@20": 0.48949,
+            "MRR@50": 0.49217,
+            "MRR@100": 0.49262,
+            "MRR@200": 0.49294,
+            "MRR@1000": 0.49317
+        },
+        "recall_cap": {
+            "R_cap@1": 0.387,
+            "R_cap@3": 0.35191,
+            "R_cap@5": 0.33359,
+            "R_cap@10": 0.29074,
+            "R_cap@20": 0.26102,
+            "R_cap@50": 0.27139,
+            "R_cap@100": 0.30159,
+            "R_cap@200": 0.36072,
+            "R_cap@1000": 0.60393
+        },
+        "hole": {
+            "Hole@1": 0.08359,
+            "Hole@3": 0.08153,
+            "Hole@5": 0.08607,
+            "Hole@10": 0.10588,
+            "Hole@20": 0.11254,
+            "Hole@50": 0.12297,
+            "Hole@100": 0.12409,
+            "Hole@200": 0.12789,
+            "Hole@1000": 0.13433
+        }
+    }
+}

eval_output/checkpoint-100000/nq.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.13789,0.20975,0.24822,0.28918,0.32125,0.3507,0.36448,0.37301,0.38303
+map,0.12029,0.18428,0.20611,0.22412,0.23357,0.23878,0.24015,0.24064,0.24091
+precision,0.13789,0.10187,0.08279,0.05614,0.03539,0.01756,0.0098,0.00526,0.00116
+recall,0.12029,0.26318,0.35262,0.47301,0.59311,0.73334,0.81301,0.87032,0.95131
+mrr,0.13789,0.20471,0.22654,0.24257,0.25099,0.25549,0.25655,0.25695,0.25715
+recall_cap,0.13789,0.26328,0.35262,0.47301,0.59311,0.73334,0.81301,0.87032,0.95131
+hole,0.85863,0.89484,0.91385,0.94056,0.9612,0.97919,0.98709,0.99174,0.99609

eval_output/checkpoint-100000/nq.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "nq",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.13789,
+            "NDCG@3": 0.20975,
+            "NDCG@5": 0.24822,
+            "NDCG@10": 0.28918,
+            "NDCG@20": 0.32125,
+            "NDCG@50": 0.3507,
+            "NDCG@100": 0.36448,
+            "NDCG@200": 0.37301,
+            "NDCG@1000": 0.38303
+        },
+        "map": {
+            "MAP@1": 0.12029,
+            "MAP@3": 0.18428,
+            "MAP@5": 0.20611,
+            "MAP@10": 0.22412,
+            "MAP@20": 0.23357,
+            "MAP@50": 0.23878,
+            "MAP@100": 0.24015,
+            "MAP@200": 0.24064,
+            "MAP@1000": 0.24091
+        },
+        "precision": {
+            "P@1": 0.13789,
+            "P@3": 0.10187,
+            "P@5": 0.08279,
+            "P@10": 0.05614,
+            "P@20": 0.03539,
+            "P@50": 0.01756,
+            "P@100": 0.0098,
+            "P@200": 0.00526,
+            "P@1000": 0.00116
+        },
+        "recall": {
+            "Recall@1": 0.12029,
+            "Recall@3": 0.26318,
+            "Recall@5": 0.35262,
+            "Recall@10": 0.47301,
+            "Recall@20": 0.59311,
+            "Recall@50": 0.73334,
+            "Recall@100": 0.81301,
+            "Recall@200": 0.87032,
+            "Recall@1000": 0.95131
+        },
+        "mrr": {
+            "MRR@1": 0.13789,
+            "MRR@3": 0.20471,
+            "MRR@5": 0.22654,
+            "MRR@10": 0.24257,
+            "MRR@20": 0.25099,
+            "MRR@50": 0.25549,
+            "MRR@100": 0.25655,
+            "MRR@200": 0.25695,
+            "MRR@1000": 0.25715
+        },
+        "recall_cap": {
+            "R_cap@1": 0.13789,
+            "R_cap@3": 0.26328,
+            "R_cap@5": 0.35262,
+            "R_cap@10": 0.47301,
+            "R_cap@20": 0.59311,
+            "R_cap@50": 0.73334,
+            "R_cap@100": 0.81301,
+            "R_cap@200": 0.87032,
+            "R_cap@1000": 0.95131
+        },
+        "hole": {
+            "Hole@1": 0.85863,
+            "Hole@3": 0.89484,
+            "Hole@5": 0.91385,
+            "Hole@10": 0.94056,
+            "Hole@20": 0.9612,
+            "Hole@50": 0.97919,
+            "Hole@100": 0.98709,
+            "Hole@200": 0.99174,
+            "Hole@1000": 0.99609
+        }
+    }
+}

eval_output/checkpoint-100000/quora.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.6863,0.74425,0.7668,0.78744,0.799,0.80727,0.80986,0.81162,0.8131
+map,0.59778,0.70299,0.72323,0.73668,0.74201,0.74452,0.74503,0.74523,0.74533
+precision,0.6863,0.32663,0.21894,0.12262,0.0665,0.02861,0.01476,0.00756,0.00156
+recall,0.59778,0.77735,0.83657,0.89623,0.93488,0.9675,0.97907,0.98836,0.99717
+mrr,0.6859,0.75415,0.76519,0.77154,0.77347,0.77418,0.77427,0.77431,0.77433
+recall_cap,0.6859,0.79025,0.84164,0.89764,0.93518,0.96753,0.97907,0.98836,0.99717
+hole,0.2928,0.63867,0.74272,0.83542,0.89072,0.92872,0.94389,0.95314,0.96223

eval_output/checkpoint-100000/quora.json

@@ -0,0 +1,86 @@
+{
+    "dataset": "quora",
+    "split": "test",
+    "metric": "dot",
+    "norm_query": false,
+    "norm_doc": false,
+    "scores": {
+        "ndcg": {
+            "NDCG@1": 0.6863,
+            "NDCG@3": 0.74425,
+            "NDCG@5": 0.7668,
+            "NDCG@10": 0.78744,
+            "NDCG@20": 0.799,
+            "NDCG@50": 0.80727,
+            "NDCG@100": 0.80986,
+            "NDCG@200": 0.81162,
+            "NDCG@1000": 0.8131
+        },
+        "map": {
+            "MAP@1": 0.59778,
+            "MAP@3": 0.70299,
+            "MAP@5": 0.72323,
+            "MAP@10": 0.73668,
+            "MAP@20": 0.74201,
+            "MAP@50": 0.74452,
+            "MAP@100": 0.74503,
+            "MAP@200": 0.74523,
+            "MAP@1000": 0.74533
+        },
+        "precision": {
+            "P@1": 0.6863,
+            "P@3": 0.32663,
+            "P@5": 0.21894,
+            "P@10": 0.12262,
+            "P@20": 0.0665,
+            "P@50": 0.02861,
+            "P@100": 0.01476,
+            "P@200": 0.00756,
+            "P@1000": 0.00156
+        },
+        "recall": {
+            "Recall@1": 0.59778,
+            "Recall@3": 0.77735,
+            "Recall@5": 0.83657,
+            "Recall@10": 0.89623,
+            "Recall@20": 0.93488,
+            "Recall@50": 0.9675,
+            "Recall@100": 0.97907,
+            "Recall@200": 0.98836,
+            "Recall@1000": 0.99717
+        },
+        "mrr": {
+            "MRR@1": 0.6859,
+            "MRR@3": 0.75415,
+            "MRR@5": 0.76519,
+            "MRR@10": 0.77154,
+            "MRR@20": 0.77347,
+            "MRR@50": 0.77418,
+            "MRR@100": 0.77427,
+            "MRR@200": 0.77431,
+            "MRR@1000": 0.77433
+        },
+        "recall_cap": {
+            "R_cap@1": 0.6859,
+            "R_cap@3": 0.79025,
+            "R_cap@5": 0.84164,
+            "R_cap@10": 0.89764,
+            "R_cap@20": 0.93518,
+            "R_cap@50": 0.96753,
+            "R_cap@100": 0.97907,
+            "R_cap@200": 0.98836,
+            "R_cap@1000": 0.99717
+        },
+        "hole": {
+            "Hole@1": 0.2928,
+            "Hole@3": 0.63867,
+            "Hole@5": 0.74272,
+            "Hole@10": 0.83542,
+            "Hole@20": 0.89072,
+            "Hole@50": 0.92872,
+            "Hole@100": 0.94389,
+            "Hole@200": 0.95314,
+            "Hole@1000": 0.96223
+        }
+    }
+}

eval_output/checkpoint-100000/scidocs.csv

@@ -0,0 +1,8 @@
+metric,@1,@3,@5,@10,@20,@50,@100,@200,@1000
+ndcg,0.181,0.14224,0.12576,0.15287,0.17834,0.20556,0.22534,0.24141,0.2773
+map,0.03683,0.06302,0.07547,0.08755,0.09599,0.1019,0.10468,0.10598,0.10736
+precision,0.181,0.131,0.1098,0.0795,0.0544,0.0298,0.01846,0.0109,0.0031
+recall,0.03683,0.07998,0.11153,0.16118,0.22043,0.30205,0.37422,0.4427,0.6287
+mrr,0.181,0.239,0.25555,0.26985,0.2777,0.28135,0.2823,0.28278,0.28309
+recall_cap,0.181,0.131,0.11153,0.16118,0.22043,0.30205,0.37422,0.4427,0.6287
+hole,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0