updates, charts, ir_datasetes

analysis.py

@@ -73,13 +73,20 @@ def get_words(words, importances):
 
 @st.cache_resource
 def get_model(model_name: str):
-    if model_name == "MonoT5":
-        pipe = pipeline('text2text-generation',
-                model='castorini/monot5-small-msmarco-10k',
-                tokenizer='castorini/monot5-small-msmarco-10k',
-                device='cpu')
+    if "MonoT5" in model_name:
+        if model_name == "MonoT5-Small":
+            pipe = pipeline('text2text-generation',
+                    model='castorini/monot5-small-msmarco-10k',
+                    tokenizer='castorini/monot5-small-msmarco-10k',
+                    device='cpu')
+        elif model_name == "MonoT5-3B":
+            pipe = pipeline('text2text-generation',
+                    model='castorini/monot5-3b-msmarco-10k',
+                    tokenizer='castorini/monot5-3b-msmarco-10k',
+                    device='cpu')
         def formatter(query, doc):
             return f"Query: {query} Document: {doc} Relevant:"
+    
         
     return pipe, formatter
 

app.py

@@ -63,7 +63,18 @@ def combine(text_og, text_new, combine_type):
 
 with st.sidebar:
     st.title("Options")
-    dataset_name = st.selectbox("Select a preloaded dataset or upload your own", tuple(ALL_DATASETS))
+    dataset_name = st.selectbox("Select a preloaded dataset or upload your own (note: some datasets are large/slow)", tuple(ALL_DATASETS))
+    if st.checkbox("Choose fields (applies to IR_Datasets only)"):
+        input_fields_doc = st.text_input("Type the name of the doc fields to get, with commas (blank=all)")
+        if input_fields_doc in ["", None]:
+            input_fields_doc = None
+        input_fields_query = st.sidebar.text_input("Type the name of the query fields to get, with commas (blank=all)")
+        if input_fields_query in ["", None]:
+            input_fields_query = None
+    else:
+        input_fields_doc = None
+        input_fields_query = None
+            
     metric_name = st.selectbox("Select a metric", tuple(ALL_METRICS))
 
     if dataset_name == "custom":
@@ -81,22 +92,24 @@ with st.sidebar:
         queries = None
         corpus = None
 
-    # sliderbar of how many Top N to choose
-    top_n = st.slider("Top N", 1, 100, 3)
+
     x = st.header('Upload a run file')
     run1_file = st.file_uploader("Choose a file", key="run1")
     y = st.header("Upload a second run file")
     run2_file = st.file_uploader("Choose a file", key="run2")
 
     z = st.header("Analysis Options")
+    # sliderbar of how many Top N to choose
+    top_n = st.slider("Top N Ranked Docs", 1, 100, 3)
+    n_relevant_docs = st.slider("Number of relevant docs", 1, 100, 3)
     incorrect_only = st.checkbox("Show only incorrect instances", value=False)
     one_better_than_two = st.checkbox("Show only instances where run 1 is better than run 2", value=False)
     two_better_than_one = st.checkbox("Show only instances where run 2 is better than run 1", value=False)
     use_model_saliency = st.checkbox("Use model saliency (slow!)", value=False)
     if use_model_saliency:
         # choose from a list of models
-        model_name = st.selectbox("Choose from a list of models", ["MonoT5"])
-        model, formatter = get_model("MonoT5")
+        model_name = st.selectbox("Choose from a list of models", ["MonoT5-Small", "MonoT5-3B"])
+        model, formatter = get_model(model_name)
         get_saliency = prep_func(model, formatter)
 
 
@@ -150,11 +163,12 @@ if check_valid_args(run1_file, run2_file, dataset_name, qrels, queries, corpus):
     # init_title = st.title("Analysis")
     # don't load these til a run is given
     if dataset_name != "custom":
-        corpus, queries, qrels = get_dataset(dataset_name)
+        corpus, queries, qrels = get_dataset(dataset_name, input_fields_doc, input_fields_query)
 
     evaluator = pytrec_eval.RelevanceEvaluator(
             copy.deepcopy(qrels), pytrec_eval.supported_measures)
     results1 = evaluator.evaluate(run1) # dict of instance then metrics then values
+    average_run1_score = pytrec_eval.compute_aggregated_measure(metric_name, [query_measures[metric_name] for query_measures in results1.values()])
     if len(results1) == 0:
         # alert and stop
         st.error("Run file is empty")
@@ -166,6 +180,8 @@ if check_valid_args(run1_file, run2_file, dataset_name, qrels, queries, corpus):
         evaluator2 = pytrec_eval.RelevanceEvaluator(
             copy.deepcopy(qrels), pytrec_eval.supported_measures)
         results2 = evaluator2.evaluate(run2)
+        average_run2_score = pytrec_eval.compute_aggregated_measure(metric_name, [query_measures[metric_name] for query_measures in results2.values()])
+
 
     col1, col2 = st.columns([1, 3], gap="large")
 
@@ -242,17 +258,29 @@ if check_valid_args(run1_file, run2_file, dataset_name, qrels, queries, corpus):
                 st.session_state.selectbox_instance = name_of_columns[st.session_state.number_of_col]
 
 
-        number_of_col = container_for_nav.number_input(min_value=-1, step=1, max_value=len(instances_to_use), on_change=sync_from_number, label=f"Select instance by index (out of **{len(instances_to_use)}**)", key="number_of_col")
+        number_of_col = container_for_nav.number_input(min_value=-1, step=1, max_value=len(instances_to_use) - 1, on_change=sync_from_number, label=f"Select instance by index (up to **{len(instances_to_use) - 1}**)", key="number_of_col")
         selectbox_instance = container_for_nav.selectbox("Select instance by ID", ["Overview"] + name_of_columns, on_change=sync_from_drop, key="selectbox_instance")
         st.divider()  
         # make pie plot showing incorrect vs correct
         st.header("Breakdown")
+
+
         if run2_file is None:
+            overall_scores_container = st.container()
+            left_score, right_score = overall_scores_container.columns([1, 1])
+            left_score.metric(label=f"Run 1 {metric_name}", value=round(average_run1_score, 3))
+            right_score.metric(label="#Q", value=len(results1))
+
             plotly_pie_chart = px.pie(names=["Perfect", "Inbetween", "None"], values=[run1_details["perfect"], run1_details["inbetween"], run1_details["none"]])
             st.write("Run 1 Scores")
             plotly_pie_chart.update_traces(showlegend=False, selector=dict(type='pie'), textposition='inside', textinfo='percent+label')
             st.plotly_chart(plotly_pie_chart, use_container_width=True)
         else:
+            overall_scores_container = st.container()
+            left_score, right_score = overall_scores_container.columns([1, 1])
+            left_score.metric(label=f"Run 1 {metric_name}", value=round(average_run1_score, 3))
+            right_score.metric(label=f"Run 2 {metric_name}", value=round(average_run2_score, 3))
+
             if st.checkbox("Show Run 1 vs Run 2", value=True):
                 plotly_pie_chart = px.pie(names=["Run 1 Better", "Run 2 Better", "Tied"], values=[is_better_run1_count, is_better_run2_count, is_same_count])
                 plotly_pie_chart.update_traces(showlegend=False, selector=dict(type='pie'), textposition='inside', textinfo='percent+label')
@@ -307,8 +335,8 @@ if check_valid_args(run1_file, run2_file, dataset_name, qrels, queries, corpus):
 
                 ## Documents
                 # relevant
-                relevant_docs = list(qrels[str(inst_num)].keys())
-                doc_texts = [(doc_id, corpus[doc_id]["title"], corpus[doc_id]["text"]) for doc_id in relevant_docs]
+                relevant_docs = list(qrels[str(inst_num)].keys())[:n_relevant_docs]
+                doc_texts = [(doc_id, corpus[doc_id]["title"] if "title" in corpus[doc_id] else "", corpus[doc_id]["text"]) for doc_id in relevant_docs]
                 st.subheader("Relevant Documents")
                 if doc_expansion1 is not None and run1_uses_doc_expansion != "None":
                     show_orig_rel = st.checkbox("Show Original Relevant Doc(s)", key=f"{inst_index}relorig", value=False)
@@ -328,14 +356,22 @@ if check_valid_args(run1_file, run2_file, dataset_name, qrels, queries, corpus):
                         st.text_area(f"{docid}:", text)
 
 
-                pred_doc = run1_pandas[run1_pandas.doc_id.isin(relevant_docs)]
-                rank_pred = pred_doc[pred_doc.qid == str(inst_num)]["rank"].tolist()
+                # go through each of the relevant documents
+                ranks = []
+                for docid in relevant_docs:
+                    pred_doc = run1_pandas[run1_pandas.doc_id.isin([docid])]
+                    rank_pred = pred_doc[pred_doc.qid == str(inst_num)]
+                    if rank_pred.empty:
+                        ranks.append("-")
+                    else:
+                        ranks.append(rank_pred.iloc[0]["rank"])
                 # st.subheader("Ranked of Documents")
                 # st.markdown(f"Rank: {rank_pred}")
-                ranking_str = ",".join([str(item) for item in rank_pred]) if type(rank_pred) == list else str(rank_pred)
+                ranking_str = ",".join([str(item) for item in ranks])
                 if ranking_str == "":
-                    ranking_str = "--"
+                    ranking_str = "-"
                 rank_col.metric(f"Rank of Relevant Doc(s)", ranking_str)
+                # breakpoint()
 
 
                 st.divider()
@@ -446,8 +482,8 @@ if check_valid_args(run1_file, run2_file, dataset_name, qrels, queries, corpus):
                 st.subheader("Relevant Documents")
                 container_two_docs_rel = st.container()
                 col_run1, col_run2 = container_two_docs_rel.columns(2, gap="medium")
-                relevant_docs = list(qrels[str(inst_num)].keys())
-                doc_texts = [(doc_id, corpus[doc_id]["title"], corpus[doc_id]["text"]) for doc_id in relevant_docs]
+                relevant_docs = list(qrels[str(inst_num)].keys())[:n_relevant_docs]
+                doc_texts = [(doc_id, corpus[doc_id]["title"] if "title" in corpus[doc_id] else "", corpus[doc_id]["text"]) for doc_id in relevant_docs]
 
                 if doc_expansion1 is not None and run1_uses_doc_expansion != "None":
                     show_orig_rel1 = col_run1.checkbox("Show Original Relevant Doc(s)", key=f"{inst_index}relorig_run1", value=False)
@@ -483,30 +519,39 @@ if check_valid_args(run1_file, run2_file, dataset_name, qrels, queries, corpus):
                 # top ranked
                 # NOTE: BEIR calls trec_eval which ranks by score, then doc_id for ties
                 # we have to fix that or we don't match the scores
-                pred_doc1 = run1_pandas[run1_pandas.qid == inst_num].sort_values(["score", "doc_id"], ascending=[False, False])
-                pred_doc1["rank_real"] = list(range(1, len(pred_doc1) + 1))
-                rank_pred1 = pred_doc1[pred_doc1.doc_id.isin(relevant_docs)]["rank_real"].tolist()
-                
-                pred_doc2 = run2_pandas[run2_pandas.qid == inst_num].sort_values(["score", "doc_id"], ascending=[False, False])
-                pred_doc2["rank_real"] = list(range(1, len(pred_doc2) + 1))
-                rank_pred2 = pred_doc2[pred_doc2.doc_id.isin(relevant_docs)]["rank_real"].tolist()
-
 
+                ranks2 = []
+                for docid in relevant_docs:
+                    pred_doc = run2_pandas[run2_pandas.doc_id.isin([docid])]
+                    rank_pred = pred_doc[pred_doc.qid == str(inst_num)]
+                    if rank_pred.empty:
+                        ranks2.append("-")
+                    else:
+                        ranks2.append(rank_pred.iloc[0]["rank"])
                 # st.subheader("Ranked of Documents")
-                # st.markdown(f"Run 1 Rank: {rank_pred1}")
-                # st.markdown(f"Run 2 Rank: {rank_pred2}")
-
-                ranking_str = ",".join([str(item) for item in rank_pred1]) if type(rank_pred1) == list else str(rank_pred1)
-                if ranking_str == "":
-                    ranking_str = "--"
-                rank_col1.metric("Run 1 " + f"Rank of Relevant Doc(s)", ranking_str)
-
-                ranking_str2 = ",".join([str(item) for item in rank_pred2]) if type(rank_pred2) == list else str(rank_pred2)
+                # st.markdown(f"Rank: {rank_pred}")
+                ranking_str2 = ",".join([str(item) for item in ranks2])
                 if ranking_str2 == "":
-                    ranking_str2 = "--"
+                    ranking_str2 = "-"
                 rank_col2.metric("Run 2 " + f"Rank of Relevant Doc(s)", ranking_str2)
 
 
+                ranks1 = []
+                for docid in relevant_docs:
+                    pred_doc = run1_pandas[run1_pandas.doc_id.isin([docid])]
+                    rank_pred = pred_doc[pred_doc.qid == str(inst_num)]
+                    if rank_pred.empty:
+                        ranks1.append("-")
+                    else:
+                        ranks1.append(rank_pred.iloc[0]["rank"])
+                # st.subheader("Ranked of Documents")
+                # st.markdown(f"Rank: {rank_pred}")
+                ranking_str1 = ",".join([str(item) for item in ranks1])
+                if ranking_str1 == "":
+                    ranking_str1 = "-"
+                rank_col1.metric("Run 1 " + f"Rank of Relevant Doc(s)", ranking_str1)
+
+
                 st.divider()
 
 

constants.py

@@ -1,3 +1,5 @@
+from ir_dataset_metadata import IR_DATASETS
+
 
 ALL_METRICS = [
     "ndcg_cut_10",
@@ -77,66 +79,12 @@ BEIR = [
 ]
 
 
-IR_DATASETS = [
-    "antique",
-    "aol_ia",
-    "aquaint",
-    "argsme",
-    "c4",
-    "car",
-    "clinicaltrials",
-    "clirmatrix",
-    "clueweb09",
-    "clueweb12",
-    "codec",
-    "cord19",
-    "cranfield",
-    "disks45",
-    "dpr_w100",
-    "codesearchnet",
-    "gov",
-    "gov2",
-    "highwire",
-    "istella22",
-    "kilt",
-    "lotte",
-    "medline",
-    "mmarco",
-    "mr_tydi",
-    "msmarco_document",
-    "msmarco_document_v2",
-    "msmarco_passage",
-    "msmarco_passage_v2",
-    "msmarco_qna",
-    "neumarco",
-    "nfcorpus",
-    "natural_questions",
-    "nyt",
-    "pmc",
-    "touche_image",
-    "touche",
-    "trec_arabic",
-    "trec_mandarin",
-    "trec_spanish",
-    "trec_robust04",
-    "trec_tot",
-    "tripclick",
-    "tweets2013_ia",
-    "vaswani",
-    "wapo",
-    "wikiclir",
-    "wikir",
-    "trec_fair",
-    "trec_cast",
-    "hc4",
-    "neuclir",
-    "sara",
-]
-
 LOCAL_DATASETS = [
     "gooaq_technical",
     "codesearch_py",
 ]
+
+
     
     
 ALL_DATASETS = ["", "custom"] + LOCAL_DATASETS + BEIR + IR_DATASETS

dataset_loading.py

@@ -104,8 +104,12 @@ def load_run(f_run):
     run_pandas.qid = run_pandas.qid.astype(str)
     run_pandas["rank"] = run_pandas["rank"].astype(int)
     run_pandas.score = run_pandas.score.astype(float)
-    # if run_1_alt is not None:
-        #     run_1_alt, run_1_alt_sub = load_jsonl(run_1_alt)
+    all_groups = []
+    for qid, sub_df in run_pandas.groupby("qid"):
+        sub_df.sort_values(["score", "doc_id"], ascending=[False, False])
+        sub_df["rank"] = list(range(1, len(sub_df) + 1))
+        all_groups.append(sub_df)
+    run_pandas = pd.concat(all_groups)
     return new_run, run_pandas
 
 
@@ -133,7 +137,7 @@ def load_jsonl(f):
     return did2text, sub_did2text
 
 
-@st.cache_data
+@st.cache_data(persist="disk")
 def get_beir(dataset: str):
     url = "https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets/{}.zip".format(dataset)
     out_dir = os.path.join(pathlib.Path(__file__).parent.absolute(), "datasets")
@@ -141,27 +145,53 @@ def get_beir(dataset: str):
     return GenericDataLoader(data_folder=data_path).load(split="test")
 
 
-@st.cache_data
-def get_ir_datasets(dataset_name: str):
+@st.cache_data(persist="disk")
+def get_ir_datasets(dataset_name: str, input_fields_doc: str = None, input_fields_query: str = None):
     dataset = ir_datasets.load(dataset_name)
     queries = {}
     for qid, query in dataset.queries_iter():
-        queries[qid] = query
-    # corpus = {}
-    # for doc in dataset.docs_iter():
+        if input_fields_query is None: 
+            if type(query) == str:
+                queries[qid] = query
+            else:
+                # get all fields that exist in query
+                all_fields = {field: getattr(query, field) for field in query._fields}
+                # put all fields into a single string
+                queries[qid] = " ".join([str(v) for v in all_fields.values()])
+        else:
+            all_fields = {field: getattr(query, field) for field in input_fields_query}
+            queries[qid] = " ".join([str(v) for v in all_fields.values()])
+
+    corpus = {}
+    for doc in dataset.docs_iter():
+        if input_fields_doc is None:
+            if type(doc) == str:
+                corpus[doc.doc_id] = {"text": doc}
+            else: # get all fields that exist in query
+                all_fields = {field: getattr(doc, field) for field in doc._fields}
+                corpus[doc.doc_id] = {"text": " ".join([str(v) for v in all_fields.values()])}
+        else:
+            all_fields = {field: getattr(doc, field) for field in input_fields_doc}
+            corpus[doc.doc_id] = {"text": " ".join([str(v) for v in all_fields.values()])}
+
     # return corpus, queries, qrels
-    return dataset.doc_store(), queries, dataset.qrels_dict()
+    return corpus, queries, dataset.qrels_dict()
 
 
-@st.cache_data
-def get_dataset(dataset_name: str):
+@st.cache_data(persist="disk")
+def get_dataset(dataset_name: str, input_fields_doc, input_fields_query):
+    if type(input_fields_doc) == str:
+        input_fields_doc = input_fields_doc.strip().split(",")
+    if type(input_fields_query) == str:
+        input_fields_query = input_fields_query.strip().split(",")
+
     if dataset_name == "":
         return {}, {}, {}
     
     if dataset_name in BEIR:
         return get_beir(dataset_name)
     elif dataset_name in IR_DATASETS:
-        return get_ir_datasets(dataset_name)
+        return get_ir_datasets(dataset_name, input_fields_doc, input_fields_query)
     elif dataset_name in LOCAL_DATASETS:
         base_path = f"local_datasets/{dataset_name}"
         corpus_file = open(f"{base_path}/corpus.jsonl", "r")

ir_dataset_metadata.py

@@ -0,0 +1,486 @@
+
+IR_DATASETS = [
+    "antique/test",
+    "antique/test/non-offensive",
+    "antique/train",
+    "antique/train/split200-train",
+    "antique/train/split200-valid",
+    "aol-ia",
+    "aquaint/trec-robust-2005",
+    "argsme/1.0/touche-2020-task-1/uncorrected",
+    "argsme/2020-04-01/processed/touche-2022-task-1",
+    "argsme/2020-04-01/touche-2020-task-1",
+    "argsme/2020-04-01/touche-2020-task-1/uncorrected",
+    "argsme/2020-04-01/touche-2021-task-1",
+    "beir/arguana",
+    "beir/climate-fever",
+    "beir/cqadupstack/android",
+    "beir/cqadupstack/english",
+    "beir/cqadupstack/gaming",
+    "beir/cqadupstack/gis",
+    "beir/cqadupstack/mathematica",
+    "beir/cqadupstack/physics",
+    "beir/cqadupstack/programmers",
+    "beir/cqadupstack/stats",
+    "beir/cqadupstack/tex",
+    "beir/cqadupstack/unix",
+    "beir/cqadupstack/webmasters",
+    "beir/cqadupstack/wordpress",
+    "beir/dbpedia-entity/dev",
+    "beir/dbpedia-entity/test",
+    "beir/fever/dev",
+    "beir/fever/test",
+    "beir/fever/train",
+    "beir/fiqa/dev",
+    "beir/fiqa/test",
+    "beir/fiqa/train",
+    "beir/hotpotqa/dev",
+    "beir/hotpotqa/test",
+    "beir/hotpotqa/train",
+    "beir/msmarco/dev",
+    "beir/msmarco/test",
+    "beir/msmarco/train",
+    "beir/nfcorpus/dev",
+    "beir/nfcorpus/test",
+    "beir/nfcorpus/train",
+    "beir/nq",
+    "beir/quora/dev",
+    "beir/quora/test",
+    "beir/scidocs",
+    "beir/scifact/test",
+    "beir/scifact/train",
+    "beir/trec-covid",
+    "beir/webis-touche2020",
+    "beir/webis-touche2020/v2",
+    "car/v1.5/test200",
+    "car/v1.5/train/fold0",
+    "car/v1.5/train/fold1",
+    "car/v1.5/train/fold2",
+    "car/v1.5/train/fold3",
+    "car/v1.5/train/fold4",
+    "car/v1.5/trec-y1/auto",
+    "car/v1.5/trec-y1/manual",
+    "clinicaltrials/2017/trec-pm-2017",
+    "clinicaltrials/2017/trec-pm-2018",
+    "clinicaltrials/2019/trec-pm-2019",
+    "clinicaltrials/2021/trec-ct-2021",
+    "clueweb09/catb/trec-web-2009",
+    "clueweb09/catb/trec-web-2009/diversity",
+    "clueweb09/catb/trec-web-2010",
+    "clueweb09/catb/trec-web-2010/diversity",
+    "clueweb09/catb/trec-web-2011",
+    "clueweb09/catb/trec-web-2011/diversity",
+    "clueweb09/catb/trec-web-2012",
+    "clueweb09/catb/trec-web-2012/diversity",
+    "clueweb09/en/trec-web-2009",
+    "clueweb09/en/trec-web-2009/diversity",
+    "clueweb09/en/trec-web-2010",
+    "clueweb09/en/trec-web-2010/diversity",
+    "clueweb09/en/trec-web-2011",
+    "clueweb09/en/trec-web-2011/diversity",
+    "clueweb09/en/trec-web-2012",
+    "clueweb09/en/trec-web-2012/diversity",
+    "clueweb09/trec-mq-2009",
+    "clueweb12/b13/clef-ehealth",
+    "clueweb12/b13/clef-ehealth/cs",
+    "clueweb12/b13/clef-ehealth/de",
+    "clueweb12/b13/clef-ehealth/fr",
+    "clueweb12/b13/clef-ehealth/hu",
+    "clueweb12/b13/clef-ehealth/pl",
+    "clueweb12/b13/clef-ehealth/sv",
+    "clueweb12/b13/ntcir-www-1",
+    "clueweb12/b13/ntcir-www-2",
+    "clueweb12/b13/trec-misinfo-2019",
+    "clueweb12/touche-2020-task-2",
+    "clueweb12/touche-2021-task-2",
+    "clueweb12/touche-2022-task-2",
+    "clueweb12/touche-2022-task-2/expanded-doc-t5-query",
+    "clueweb12/trec-web-2013",
+    "clueweb12/trec-web-2013/diversity",
+    "clueweb12/trec-web-2014",
+    "clueweb12/trec-web-2014/diversity",
+    "codec",
+    "codec/economics",
+    "codec/history",
+    "codec/politics",
+    "codesearchnet/challenge",
+    "codesearchnet/test",
+    "codesearchnet/train",
+    "codesearchnet/valid",
+    "cord19/fulltext/trec-covid",
+    "cord19/trec-covid",
+    "cord19/trec-covid/round1",
+    "cord19/trec-covid/round2",
+    "cord19/trec-covid/round3",
+    "cord19/trec-covid/round4",
+    "cord19/trec-covid/round5",
+    "cranfield",
+    "disks45/nocr/trec-robust-2004",
+    "disks45/nocr/trec-robust-2004/fold1",
+    "disks45/nocr/trec-robust-2004/fold2",
+    "disks45/nocr/trec-robust-2004/fold3",
+    "disks45/nocr/trec-robust-2004/fold4",
+    "disks45/nocr/trec-robust-2004/fold5",
+    "disks45/nocr/trec7",
+    "disks45/nocr/trec8",
+    "dpr-w100/natural-questions/dev",
+    "dpr-w100/natural-questions/train",
+    "dpr-w100/trivia-qa/dev",
+    "dpr-w100/trivia-qa/train",
+    "gov/trec-web-2002",
+    "gov/trec-web-2002/named-page",
+    "gov/trec-web-2003",
+    "gov/trec-web-2003/named-page",
+    "gov/trec-web-2004",
+    "gov2/trec-mq-2007",
+    "gov2/trec-mq-2008",
+    "gov2/trec-tb-2004",
+    "gov2/trec-tb-2005",
+    "gov2/trec-tb-2005/efficiency",
+    "gov2/trec-tb-2005/named-page",
+    "gov2/trec-tb-2006",
+    "gov2/trec-tb-2006/efficiency",
+    "gov2/trec-tb-2006/efficiency/stream3",
+    "gov2/trec-tb-2006/named-page",
+    "hc4/fa/dev",
+    "hc4/fa/test",
+    "hc4/fa/train",
+    "hc4/ru/dev",
+    "hc4/ru/test",
+    "hc4/ru/train",
+    "hc4/zh/dev",
+    "hc4/zh/test",
+    "hc4/zh/train",
+    "highwire/trec-genomics-2006",
+    "highwire/trec-genomics-2007",
+    "istella22/test",
+    "istella22/test/fold1",
+    "istella22/test/fold2",
+    "istella22/test/fold3",
+    "istella22/test/fold4",
+    "istella22/test/fold5",
+    "kilt/codec",
+    "kilt/codec/economics",
+    "kilt/codec/history",
+    "kilt/codec/politics",
+    "lotte/lifestyle/dev/forum",
+    "lotte/lifestyle/dev/search",
+    "lotte/lifestyle/test/forum",
+    "lotte/lifestyle/test/search",
+    "lotte/pooled/dev/forum",
+    "lotte/pooled/dev/search",
+    "lotte/pooled/test/forum",
+    "lotte/pooled/test/search",
+    "lotte/recreation/dev/forum",
+    "lotte/recreation/dev/search",
+    "lotte/recreation/test/forum",
+    "lotte/recreation/test/search",
+    "lotte/science/dev/forum",
+    "lotte/science/dev/search",
+    "lotte/science/test/forum",
+    "lotte/science/test/search",
+    "lotte/technology/dev/forum",
+    "lotte/technology/dev/search",
+    "lotte/technology/test/forum",
+    "lotte/technology/test/search",
+    "lotte/writing/dev/forum",
+    "lotte/writing/dev/search",
+    "lotte/writing/test/forum",
+    "lotte/writing/test/search",
+    "medline/2004/trec-genomics-2004",
+    "medline/2004/trec-genomics-2005",
+    "medline/2017/trec-pm-2017",
+    "medline/2017/trec-pm-2018",
+    "mmarco/de/dev",
+    "mmarco/de/dev/small",
+    "mmarco/de/train",
+    "mmarco/es/dev",
+    "mmarco/es/dev/small",
+    "mmarco/es/train",
+    "mmarco/fr/dev",
+    "mmarco/fr/dev/small",
+    "mmarco/fr/train",
+    "mmarco/id/dev",
+    "mmarco/id/dev/small",
+    "mmarco/id/train",
+    "mmarco/it/dev",
+    "mmarco/it/dev/small",
+    "mmarco/it/train",
+    "mmarco/pt/dev",
+    "mmarco/pt/dev/small",
+    "mmarco/pt/dev/small/v1.1",
+    "mmarco/pt/dev/v1.1",
+    "mmarco/pt/train",
+    "mmarco/pt/train/v1.1",
+    "mmarco/ru/dev",
+    "mmarco/ru/dev/small",
+    "mmarco/ru/train",
+    "mmarco/v2/ar/dev",
+    "mmarco/v2/ar/dev/small",
+    "mmarco/v2/ar/train",
+    "mmarco/v2/de/dev",
+    "mmarco/v2/de/dev/small",
+    "mmarco/v2/de/train",
+    "mmarco/v2/dt/dev",
+    "mmarco/v2/dt/dev/small",
+    "mmarco/v2/dt/train",
+    "mmarco/v2/es/dev",
+    "mmarco/v2/es/dev/small",
+    "mmarco/v2/es/train",
+    "mmarco/v2/fr/dev",
+    "mmarco/v2/fr/dev/small",
+    "mmarco/v2/fr/train",
+    "mmarco/v2/hi/dev",
+    "mmarco/v2/hi/dev/small",
+    "mmarco/v2/hi/train",
+    "mmarco/v2/id/dev",
+    "mmarco/v2/id/dev/small",
+    "mmarco/v2/id/train",
+    "mmarco/v2/it/dev",
+    "mmarco/v2/it/dev/small",
+    "mmarco/v2/it/train",
+    "mmarco/v2/ja/dev",
+    "mmarco/v2/ja/dev/small",
+    "mmarco/v2/ja/train",
+    "mmarco/v2/pt/dev",
+    "mmarco/v2/pt/dev/small",
+    "mmarco/v2/pt/train",
+    "mmarco/v2/ru/dev",
+    "mmarco/v2/ru/dev/small",
+    "mmarco/v2/ru/train",
+    "mmarco/v2/vi/dev",
+    "mmarco/v2/vi/dev/small",
+    "mmarco/v2/vi/train",
+    "mmarco/v2/zh/dev",
+    "mmarco/v2/zh/dev/small",
+    "mmarco/v2/zh/train",
+    "mmarco/zh/dev",
+    "mmarco/zh/dev/small",
+    "mmarco/zh/dev/small/v1.1",
+    "mmarco/zh/dev/v1.1",
+    "mmarco/zh/train",
+    "mr-tydi/ar",
+    "mr-tydi/ar/dev",
+    "mr-tydi/ar/test",
+    "mr-tydi/ar/train",
+    "mr-tydi/bn",
+    "mr-tydi/bn/dev",
+    "mr-tydi/bn/test",
+    "mr-tydi/bn/train",
+    "mr-tydi/en",
+    "mr-tydi/en/dev",
+    "mr-tydi/en/test",
+    "mr-tydi/en/train",
+    "mr-tydi/fi",
+    "mr-tydi/fi/dev",
+    "mr-tydi/fi/test",
+    "mr-tydi/fi/train",
+    "mr-tydi/id",
+    "mr-tydi/id/dev",
+    "mr-tydi/id/test",
+    "mr-tydi/id/train",
+    "mr-tydi/ja",
+    "mr-tydi/ja/dev",
+    "mr-tydi/ja/test",
+    "mr-tydi/ja/train",
+    "mr-tydi/ko",
+    "mr-tydi/ko/dev",
+    "mr-tydi/ko/test",
+    "mr-tydi/ko/train",
+    "mr-tydi/ru",
+    "mr-tydi/ru/dev",
+    "mr-tydi/ru/test",
+    "mr-tydi/ru/train",
+    "mr-tydi/sw",
+    "mr-tydi/sw/dev",
+    "mr-tydi/sw/test",
+    "mr-tydi/sw/train",
+    "mr-tydi/te",
+    "mr-tydi/te/dev",
+    "mr-tydi/te/test",
+    "mr-tydi/te/train",
+    "mr-tydi/th",
+    "mr-tydi/th/dev",
+    "mr-tydi/th/test",
+    "mr-tydi/th/train",
+    "msmarco-document-v2/dev1",
+    "msmarco-document-v2/dev2",
+    "msmarco-document-v2/train",
+    "msmarco-document-v2/trec-dl-2019",
+    "msmarco-document-v2/trec-dl-2019/judged",
+    "msmarco-document-v2/trec-dl-2020",
+    "msmarco-document-v2/trec-dl-2020/judged",
+    "msmarco-document-v2/trec-dl-2021",
+    "msmarco-document-v2/trec-dl-2021/judged",
+    "msmarco-document-v2/trec-dl-2022",
+    "msmarco-document-v2/trec-dl-2022/judged",
+    "msmarco-document/dev",
+    "msmarco-document/orcas",
+    "msmarco-document/train",
+    "msmarco-document/trec-dl-2019",
+    "msmarco-document/trec-dl-2019/judged",
+    "msmarco-document/trec-dl-2020",
+    "msmarco-document/trec-dl-2020/judged",
+    "msmarco-document/trec-dl-hard",
+    "msmarco-document/trec-dl-hard/fold1",
+    "msmarco-document/trec-dl-hard/fold2",
+    "msmarco-document/trec-dl-hard/fold3",
+    "msmarco-document/trec-dl-hard/fold4",
+    "msmarco-document/trec-dl-hard/fold5",
+    "msmarco-passage-v2/dev1",
+    "msmarco-passage-v2/dev2",
+    "msmarco-passage-v2/train",
+    "msmarco-passage-v2/trec-dl-2021",
+    "msmarco-passage-v2/trec-dl-2021/judged",
+    "msmarco-passage-v2/trec-dl-2022",
+    "msmarco-passage-v2/trec-dl-2022/judged",
+    "msmarco-passage/dev",
+    "msmarco-passage/dev/2",
+    "msmarco-passage/dev/judged",
+    "msmarco-passage/dev/small",
+    "msmarco-passage/train",
+    "msmarco-passage/train/judged",
+    "msmarco-passage/train/medical",
+    "msmarco-passage/train/split200-train",
+    "msmarco-passage/train/split200-valid",
+    "msmarco-passage/train/triples-small",
+    "msmarco-passage/train/triples-v2",
+    "msmarco-passage/trec-dl-2019",
+    "msmarco-passage/trec-dl-2019/judged",
+    "msmarco-passage/trec-dl-2020",
+    "msmarco-passage/trec-dl-2020/judged",
+    "msmarco-passage/trec-dl-hard",
+    "msmarco-passage/trec-dl-hard/fold1",
+    "msmarco-passage/trec-dl-hard/fold2",
+    "msmarco-passage/trec-dl-hard/fold3",
+    "msmarco-passage/trec-dl-hard/fold4",
+    "msmarco-passage/trec-dl-hard/fold5",
+    "msmarco-qna/dev",
+    "msmarco-qna/train",
+    "natural-questions/dev",
+    "natural-questions/train",
+    "neuclir/1/fa/hc4-filtered",
+    "neuclir/1/ru/hc4-filtered",
+    "neuclir/1/zh/hc4-filtered",
+    "neumarco/fa/dev",
+    "neumarco/fa/dev/judged",
+    "neumarco/fa/dev/small",
+    "neumarco/fa/train",
+    "neumarco/fa/train/judged",
+    "neumarco/ru/dev",
+    "neumarco/ru/dev/judged",
+    "neumarco/ru/dev/small",
+    "neumarco/ru/train",
+    "neumarco/ru/train/judged",
+    "neumarco/zh/dev",
+    "neumarco/zh/dev/judged",
+    "neumarco/zh/dev/small",
+    "neumarco/zh/train",
+    "neumarco/zh/train/judged",
+    "nfcorpus/dev",
+    "nfcorpus/dev/nontopic",
+    "nfcorpus/dev/video",
+    "nfcorpus/test",
+    "nfcorpus/test/nontopic",
+    "nfcorpus/test/video",
+    "nfcorpus/train",
+    "nfcorpus/train/nontopic",
+    "nfcorpus/train/video",
+    "nyt/trec-core-2017",
+    "nyt/wksup",
+    "nyt/wksup/train",
+    "nyt/wksup/valid",
+    "pmc/v1/trec-cds-2014",
+    "pmc/v1/trec-cds-2015",
+    "pmc/v2/trec-cds-2016",
+    "sara",
+    "touche-image/2022-06-13/touche-2022-task-3",
+    "trec-arabic/ar2001",
+    "trec-arabic/ar2002",
+    "trec-cast/v0/train",
+    "trec-cast/v0/train/judged",
+    "trec-cast/v1/2019",
+    "trec-cast/v1/2019/judged",
+    "trec-cast/v1/2020",
+    "trec-cast/v1/2020/judged",
+    "trec-fair-2021/eval",
+    "trec-fair-2021/train",
+    "trec-fair/2021/eval",
+    "trec-fair/2021/train",
+    "trec-fair/2022/train",
+    "trec-mandarin/trec5",
+    "trec-mandarin/trec6",
+    "trec-robust04",
+    "trec-robust04/fold1",
+    "trec-robust04/fold2",
+    "trec-robust04/fold3",
+    "trec-robust04/fold4",
+    "trec-robust04/fold5",
+    "trec-spanish/trec3",
+    "trec-spanish/trec4",
+    "trec-tot/2023/dev",
+    "trec-tot/2023/train",
+    "tripclick/train",
+    "tripclick/train/head",
+    "tripclick/train/head/dctr",
+    "tripclick/train/hofstaetter-triples",
+    "tripclick/train/tail",
+    "tripclick/train/torso",
+    "tripclick/val",
+    "tripclick/val/head",
+    "tripclick/val/head/dctr",
+    "tripclick/val/tail",
+    "tripclick/val/torso",
+    "tweets2013-ia/trec-mb-2013",
+    "tweets2013-ia/trec-mb-2014",
+    "vaswani",
+    "wapo/v2/trec-core-2018",
+    "wapo/v2/trec-news-2018",
+    "wapo/v2/trec-news-2019",
+    "wikiclir/ar",
+    "wikiclir/ca",
+    "wikiclir/cs",
+    "wikiclir/de",
+    "wikiclir/en-simple",
+    "wikiclir/es",
+    "wikiclir/fi",
+    "wikiclir/fr",
+    "wikiclir/it",
+    "wikiclir/ja",
+    "wikiclir/ko",
+    "wikiclir/nl",
+    "wikiclir/nn",
+    "wikiclir/no",
+    "wikiclir/pl",
+    "wikiclir/pt",
+    "wikiclir/ro",
+    "wikiclir/ru",
+    "wikiclir/sv",
+    "wikiclir/sw",
+    "wikiclir/tl",
+    "wikiclir/tr",
+    "wikiclir/uk",
+    "wikiclir/vi",
+    "wikiclir/zh",
+    "wikir/en1k/test",
+    "wikir/en1k/training",
+    "wikir/en1k/validation",
+    "wikir/en59k/test",
+    "wikir/en59k/training",
+    "wikir/en59k/validation",
+    "wikir/en78k/test",
+    "wikir/en78k/training",
+    "wikir/en78k/validation",
+    "wikir/ens78k/test",
+    "wikir/ens78k/training",
+    "wikir/ens78k/validation",
+    "wikir/es13k/test",
+    "wikir/es13k/training",
+    "wikir/es13k/validation",
+    "wikir/fr14k/test",
+    "wikir/fr14k/training",
+    "wikir/fr14k/validation",
+    "wikir/it16k/test",
+    "wikir/it16k/training",
+    "wikir/it16k/validation"
+]

ir_dataset_names.json

@@ -0,0 +1,485 @@
+[
+    "antique/test",
+    "antique/test/non-offensive",
+    "antique/train",
+    "antique/train/split200-train",
+    "antique/train/split200-valid",
+    "aol-ia",
+    "aquaint/trec-robust-2005",
+    "argsme/1.0/touche-2020-task-1/uncorrected",
+    "argsme/2020-04-01/processed/touche-2022-task-1",
+    "argsme/2020-04-01/touche-2020-task-1",
+    "argsme/2020-04-01/touche-2020-task-1/uncorrected",
+    "argsme/2020-04-01/touche-2021-task-1",
+    "beir/arguana",
+    "beir/climate-fever",
+    "beir/cqadupstack/android",
+    "beir/cqadupstack/english",
+    "beir/cqadupstack/gaming",
+    "beir/cqadupstack/gis",
+    "beir/cqadupstack/mathematica",
+    "beir/cqadupstack/physics",
+    "beir/cqadupstack/programmers",
+    "beir/cqadupstack/stats",
+    "beir/cqadupstack/tex",
+    "beir/cqadupstack/unix",
+    "beir/cqadupstack/webmasters",
+    "beir/cqadupstack/wordpress",
+    "beir/dbpedia-entity/dev",
+    "beir/dbpedia-entity/test",
+    "beir/fever/dev",
+    "beir/fever/test",
+    "beir/fever/train",
+    "beir/fiqa/dev",
+    "beir/fiqa/test",
+    "beir/fiqa/train",
+    "beir/hotpotqa/dev",
+    "beir/hotpotqa/test",
+    "beir/hotpotqa/train",
+    "beir/msmarco/dev",
+    "beir/msmarco/test",
+    "beir/msmarco/train",
+    "beir/nfcorpus/dev",
+    "beir/nfcorpus/test",
+    "beir/nfcorpus/train",
+    "beir/nq",
+    "beir/quora/dev",
+    "beir/quora/test",
+    "beir/scidocs",
+    "beir/scifact/test",
+    "beir/scifact/train",
+    "beir/trec-covid",
+    "beir/webis-touche2020",
+    "beir/webis-touche2020/v2",
+    "car/v1.5/test200",
+    "car/v1.5/train/fold0",
+    "car/v1.5/train/fold1",
+    "car/v1.5/train/fold2",
+    "car/v1.5/train/fold3",
+    "car/v1.5/train/fold4",
+    "car/v1.5/trec-y1/auto",
+    "car/v1.5/trec-y1/manual",
+    "clinicaltrials/2017/trec-pm-2017",
+    "clinicaltrials/2017/trec-pm-2018",
+    "clinicaltrials/2019/trec-pm-2019",
+    "clinicaltrials/2021/trec-ct-2021",
+    "clueweb09/catb/trec-web-2009",
+    "clueweb09/catb/trec-web-2009/diversity",
+    "clueweb09/catb/trec-web-2010",
+    "clueweb09/catb/trec-web-2010/diversity",
+    "clueweb09/catb/trec-web-2011",
+    "clueweb09/catb/trec-web-2011/diversity",
+    "clueweb09/catb/trec-web-2012",
+    "clueweb09/catb/trec-web-2012/diversity",
+    "clueweb09/en/trec-web-2009",
+    "clueweb09/en/trec-web-2009/diversity",
+    "clueweb09/en/trec-web-2010",
+    "clueweb09/en/trec-web-2010/diversity",
+    "clueweb09/en/trec-web-2011",
+    "clueweb09/en/trec-web-2011/diversity",
+    "clueweb09/en/trec-web-2012",
+    "clueweb09/en/trec-web-2012/diversity",
+    "clueweb09/trec-mq-2009",
+    "clueweb12/b13/clef-ehealth",
+    "clueweb12/b13/clef-ehealth/cs",
+    "clueweb12/b13/clef-ehealth/de",
+    "clueweb12/b13/clef-ehealth/fr",
+    "clueweb12/b13/clef-ehealth/hu",
+    "clueweb12/b13/clef-ehealth/pl",
+    "clueweb12/b13/clef-ehealth/sv",
+    "clueweb12/b13/ntcir-www-1",
+    "clueweb12/b13/ntcir-www-2",
+    "clueweb12/b13/trec-misinfo-2019",
+    "clueweb12/touche-2020-task-2",
+    "clueweb12/touche-2021-task-2",
+    "clueweb12/touche-2022-task-2",
+    "clueweb12/touche-2022-task-2/expanded-doc-t5-query",
+    "clueweb12/trec-web-2013",
+    "clueweb12/trec-web-2013/diversity",
+    "clueweb12/trec-web-2014",
+    "clueweb12/trec-web-2014/diversity",
+    "codec",
+    "codec/economics",
+    "codec/history",
+    "codec/politics",
+    "codesearchnet/challenge",
+    "codesearchnet/test",
+    "codesearchnet/train",
+    "codesearchnet/valid",
+    "cord19/fulltext/trec-covid",
+    "cord19/trec-covid",
+    "cord19/trec-covid/round1",
+    "cord19/trec-covid/round2",
+    "cord19/trec-covid/round3",
+    "cord19/trec-covid/round4",
+    "cord19/trec-covid/round5",
+    "cranfield",
+    "disks45/nocr/trec-robust-2004",
+    "disks45/nocr/trec-robust-2004/fold1",
+    "disks45/nocr/trec-robust-2004/fold2",
+    "disks45/nocr/trec-robust-2004/fold3",
+    "disks45/nocr/trec-robust-2004/fold4",
+    "disks45/nocr/trec-robust-2004/fold5",
+    "disks45/nocr/trec7",
+    "disks45/nocr/trec8",
+    "dpr-w100/natural-questions/dev",
+    "dpr-w100/natural-questions/train",
+    "dpr-w100/trivia-qa/dev",
+    "dpr-w100/trivia-qa/train",
+    "gov/trec-web-2002",
+    "gov/trec-web-2002/named-page",
+    "gov/trec-web-2003",
+    "gov/trec-web-2003/named-page",
+    "gov/trec-web-2004",
+    "gov2/trec-mq-2007",
+    "gov2/trec-mq-2008",
+    "gov2/trec-tb-2004",
+    "gov2/trec-tb-2005",
+    "gov2/trec-tb-2005/efficiency",
+    "gov2/trec-tb-2005/named-page",
+    "gov2/trec-tb-2006",
+    "gov2/trec-tb-2006/efficiency",
+    "gov2/trec-tb-2006/efficiency/stream3",
+    "gov2/trec-tb-2006/named-page",
+    "hc4/fa/dev",
+    "hc4/fa/test",
+    "hc4/fa/train",
+    "hc4/ru/dev",
+    "hc4/ru/test",
+    "hc4/ru/train",
+    "hc4/zh/dev",
+    "hc4/zh/test",
+    "hc4/zh/train",
+    "highwire/trec-genomics-2006",
+    "highwire/trec-genomics-2007",
+    "istella22/test",
+    "istella22/test/fold1",
+    "istella22/test/fold2",
+    "istella22/test/fold3",
+    "istella22/test/fold4",
+    "istella22/test/fold5",
+    "kilt/codec",
+    "kilt/codec/economics",
+    "kilt/codec/history",
+    "kilt/codec/politics",
+    "lotte/lifestyle/dev/forum",
+    "lotte/lifestyle/dev/search",
+    "lotte/lifestyle/test/forum",
+    "lotte/lifestyle/test/search",
+    "lotte/pooled/dev/forum",
+    "lotte/pooled/dev/search",
+    "lotte/pooled/test/forum",
+    "lotte/pooled/test/search",
+    "lotte/recreation/dev/forum",
+    "lotte/recreation/dev/search",
+    "lotte/recreation/test/forum",
+    "lotte/recreation/test/search",
+    "lotte/science/dev/forum",
+    "lotte/science/dev/search",
+    "lotte/science/test/forum",
+    "lotte/science/test/search",
+    "lotte/technology/dev/forum",
+    "lotte/technology/dev/search",
+    "lotte/technology/test/forum",
+    "lotte/technology/test/search",
+    "lotte/writing/dev/forum",
+    "lotte/writing/dev/search",
+    "lotte/writing/test/forum",
+    "lotte/writing/test/search",
+    "medline/2004/trec-genomics-2004",
+    "medline/2004/trec-genomics-2005",
+    "medline/2017/trec-pm-2017",
+    "medline/2017/trec-pm-2018",
+    "mmarco/de/dev",
+    "mmarco/de/dev/small",
+    "mmarco/de/train",
+    "mmarco/es/dev",
+    "mmarco/es/dev/small",
+    "mmarco/es/train",
+    "mmarco/fr/dev",
+    "mmarco/fr/dev/small",
+    "mmarco/fr/train",
+    "mmarco/id/dev",
+    "mmarco/id/dev/small",
+    "mmarco/id/train",
+    "mmarco/it/dev",
+    "mmarco/it/dev/small",
+    "mmarco/it/train",
+    "mmarco/pt/dev",
+    "mmarco/pt/dev/small",
+    "mmarco/pt/dev/small/v1.1",
+    "mmarco/pt/dev/v1.1",
+    "mmarco/pt/train",
+    "mmarco/pt/train/v1.1",
+    "mmarco/ru/dev",
+    "mmarco/ru/dev/small",
+    "mmarco/ru/train",
+    "mmarco/v2/ar/dev",
+    "mmarco/v2/ar/dev/small",
+    "mmarco/v2/ar/train",
+    "mmarco/v2/de/dev",
+    "mmarco/v2/de/dev/small",
+    "mmarco/v2/de/train",
+    "mmarco/v2/dt/dev",
+    "mmarco/v2/dt/dev/small",
+    "mmarco/v2/dt/train",
+    "mmarco/v2/es/dev",
+    "mmarco/v2/es/dev/small",
+    "mmarco/v2/es/train",
+    "mmarco/v2/fr/dev",
+    "mmarco/v2/fr/dev/small",
+    "mmarco/v2/fr/train",
+    "mmarco/v2/hi/dev",
+    "mmarco/v2/hi/dev/small",
+    "mmarco/v2/hi/train",
+    "mmarco/v2/id/dev",
+    "mmarco/v2/id/dev/small",
+    "mmarco/v2/id/train",
+    "mmarco/v2/it/dev",
+    "mmarco/v2/it/dev/small",
+    "mmarco/v2/it/train",
+    "mmarco/v2/ja/dev",
+    "mmarco/v2/ja/dev/small",
+    "mmarco/v2/ja/train",
+    "mmarco/v2/pt/dev",
+    "mmarco/v2/pt/dev/small",
+    "mmarco/v2/pt/train",
+    "mmarco/v2/ru/dev",
+    "mmarco/v2/ru/dev/small",
+    "mmarco/v2/ru/train",
+    "mmarco/v2/vi/dev",
+    "mmarco/v2/vi/dev/small",
+    "mmarco/v2/vi/train",
+    "mmarco/v2/zh/dev",
+    "mmarco/v2/zh/dev/small",
+    "mmarco/v2/zh/train",
+    "mmarco/zh/dev",
+    "mmarco/zh/dev/small",
+    "mmarco/zh/dev/small/v1.1",
+    "mmarco/zh/dev/v1.1",
+    "mmarco/zh/train",
+    "mr-tydi/ar",
+    "mr-tydi/ar/dev",
+    "mr-tydi/ar/test",
+    "mr-tydi/ar/train",
+    "mr-tydi/bn",
+    "mr-tydi/bn/dev",
+    "mr-tydi/bn/test",
+    "mr-tydi/bn/train",
+    "mr-tydi/en",
+    "mr-tydi/en/dev",
+    "mr-tydi/en/test",
+    "mr-tydi/en/train",
+    "mr-tydi/fi",
+    "mr-tydi/fi/dev",
+    "mr-tydi/fi/test",
+    "mr-tydi/fi/train",
+    "mr-tydi/id",
+    "mr-tydi/id/dev",
+    "mr-tydi/id/test",
+    "mr-tydi/id/train",
+    "mr-tydi/ja",
+    "mr-tydi/ja/dev",
+    "mr-tydi/ja/test",
+    "mr-tydi/ja/train",
+    "mr-tydi/ko",
+    "mr-tydi/ko/dev",
+    "mr-tydi/ko/test",
+    "mr-tydi/ko/train",
+    "mr-tydi/ru",
+    "mr-tydi/ru/dev",
+    "mr-tydi/ru/test",
+    "mr-tydi/ru/train",
+    "mr-tydi/sw",
+    "mr-tydi/sw/dev",
+    "mr-tydi/sw/test",
+    "mr-tydi/sw/train",
+    "mr-tydi/te",
+    "mr-tydi/te/dev",
+    "mr-tydi/te/test",
+    "mr-tydi/te/train",
+    "mr-tydi/th",
+    "mr-tydi/th/dev",
+    "mr-tydi/th/test",
+    "mr-tydi/th/train",
+    "msmarco-document-v2/dev1",
+    "msmarco-document-v2/dev2",
+    "msmarco-document-v2/train",
+    "msmarco-document-v2/trec-dl-2019",
+    "msmarco-document-v2/trec-dl-2019/judged",
+    "msmarco-document-v2/trec-dl-2020",
+    "msmarco-document-v2/trec-dl-2020/judged",
+    "msmarco-document-v2/trec-dl-2021",
+    "msmarco-document-v2/trec-dl-2021/judged",
+    "msmarco-document-v2/trec-dl-2022",
+    "msmarco-document-v2/trec-dl-2022/judged",
+    "msmarco-document/dev",
+    "msmarco-document/orcas",
+    "msmarco-document/train",
+    "msmarco-document/trec-dl-2019",
+    "msmarco-document/trec-dl-2019/judged",
+    "msmarco-document/trec-dl-2020",
+    "msmarco-document/trec-dl-2020/judged",
+    "msmarco-document/trec-dl-hard",
+    "msmarco-document/trec-dl-hard/fold1",
+    "msmarco-document/trec-dl-hard/fold2",
+    "msmarco-document/trec-dl-hard/fold3",
+    "msmarco-document/trec-dl-hard/fold4",
+    "msmarco-document/trec-dl-hard/fold5",
+    "msmarco-passage-v2/dev1",
+    "msmarco-passage-v2/dev2",
+    "msmarco-passage-v2/train",
+    "msmarco-passage-v2/trec-dl-2021",
+    "msmarco-passage-v2/trec-dl-2021/judged",
+    "msmarco-passage-v2/trec-dl-2022",
+    "msmarco-passage-v2/trec-dl-2022/judged",
+    "msmarco-passage/dev",
+    "msmarco-passage/dev/2",
+    "msmarco-passage/dev/judged",
+    "msmarco-passage/dev/small",
+    "msmarco-passage/train",
+    "msmarco-passage/train/judged",
+    "msmarco-passage/train/medical",
+    "msmarco-passage/train/split200-train",
+    "msmarco-passage/train/split200-valid",
+    "msmarco-passage/train/triples-small",
+    "msmarco-passage/train/triples-v2",
+    "msmarco-passage/trec-dl-2019",
+    "msmarco-passage/trec-dl-2019/judged",
+    "msmarco-passage/trec-dl-2020",
+    "msmarco-passage/trec-dl-2020/judged",
+    "msmarco-passage/trec-dl-hard",
+    "msmarco-passage/trec-dl-hard/fold1",
+    "msmarco-passage/trec-dl-hard/fold2",
+    "msmarco-passage/trec-dl-hard/fold3",
+    "msmarco-passage/trec-dl-hard/fold4",
+    "msmarco-passage/trec-dl-hard/fold5",
+    "msmarco-qna/dev",
+    "msmarco-qna/train",
+    "natural-questions/dev",
+    "natural-questions/train",
+    "neuclir/1/fa/hc4-filtered",
+    "neuclir/1/ru/hc4-filtered",
+    "neuclir/1/zh/hc4-filtered",
+    "neumarco/fa/dev",
+    "neumarco/fa/dev/judged",
+    "neumarco/fa/dev/small",
+    "neumarco/fa/train",
+    "neumarco/fa/train/judged",
+    "neumarco/ru/dev",
+    "neumarco/ru/dev/judged",
+    "neumarco/ru/dev/small",
+    "neumarco/ru/train",
+    "neumarco/ru/train/judged",
+    "neumarco/zh/dev",
+    "neumarco/zh/dev/judged",
+    "neumarco/zh/dev/small",
+    "neumarco/zh/train",
+    "neumarco/zh/train/judged",
+    "nfcorpus/dev",
+    "nfcorpus/dev/nontopic",
+    "nfcorpus/dev/video",
+    "nfcorpus/test",
+    "nfcorpus/test/nontopic",
+    "nfcorpus/test/video",
+    "nfcorpus/train",
+    "nfcorpus/train/nontopic",
+    "nfcorpus/train/video",
+    "nyt/trec-core-2017",
+    "nyt/wksup",
+    "nyt/wksup/train",
+    "nyt/wksup/valid",
+    "pmc/v1/trec-cds-2014",
+    "pmc/v1/trec-cds-2015",
+    "pmc/v2/trec-cds-2016",
+    "sara",
+    "touche-image/2022-06-13/touche-2022-task-3",
+    "trec-arabic/ar2001",
+    "trec-arabic/ar2002",
+    "trec-cast/v0/train",
+    "trec-cast/v0/train/judged",
+    "trec-cast/v1/2019",
+    "trec-cast/v1/2019/judged",
+    "trec-cast/v1/2020",
+    "trec-cast/v1/2020/judged",
+    "trec-fair-2021/eval",
+    "trec-fair-2021/train",
+    "trec-fair/2021/eval",
+    "trec-fair/2021/train",
+    "trec-fair/2022/train",
+    "trec-mandarin/trec5",
+    "trec-mandarin/trec6",
+    "trec-robust04",
+    "trec-robust04/fold1",
+    "trec-robust04/fold2",
+    "trec-robust04/fold3",
+    "trec-robust04/fold4",
+    "trec-robust04/fold5",
+    "trec-spanish/trec3",
+    "trec-spanish/trec4",
+    "trec-tot/2023/dev",
+    "trec-tot/2023/train",
+    "tripclick/train",
+    "tripclick/train/head",
+    "tripclick/train/head/dctr",
+    "tripclick/train/hofstaetter-triples",
+    "tripclick/train/tail",
+    "tripclick/train/torso",
+    "tripclick/val",
+    "tripclick/val/head",
+    "tripclick/val/head/dctr",
+    "tripclick/val/tail",
+    "tripclick/val/torso",
+    "tweets2013-ia/trec-mb-2013",
+    "tweets2013-ia/trec-mb-2014",
+    "vaswani",
+    "wapo/v2/trec-core-2018",
+    "wapo/v2/trec-news-2018",
+    "wapo/v2/trec-news-2019",
+    "wikiclir/ar",
+    "wikiclir/ca",
+    "wikiclir/cs",
+    "wikiclir/de",
+    "wikiclir/en-simple",
+    "wikiclir/es",
+    "wikiclir/fi",
+    "wikiclir/fr",
+    "wikiclir/it",
+    "wikiclir/ja",
+    "wikiclir/ko",
+    "wikiclir/nl",
+    "wikiclir/nn",
+    "wikiclir/no",
+    "wikiclir/pl",
+    "wikiclir/pt",
+    "wikiclir/ro",
+    "wikiclir/ru",
+    "wikiclir/sv",
+    "wikiclir/sw",
+    "wikiclir/tl",
+    "wikiclir/tr",
+    "wikiclir/uk",
+    "wikiclir/vi",
+    "wikiclir/zh",
+    "wikir/en1k/test",
+    "wikir/en1k/training",
+    "wikir/en1k/validation",
+    "wikir/en59k/test",
+    "wikir/en59k/training",
+    "wikir/en59k/validation",
+    "wikir/en78k/test",
+    "wikir/en78k/training",
+    "wikir/en78k/validation",
+    "wikir/ens78k/test",
+    "wikir/ens78k/training",
+    "wikir/ens78k/validation",
+    "wikir/es13k/test",
+    "wikir/es13k/training",
+    "wikir/es13k/validation",
+    "wikir/fr14k/test",
+    "wikir/fr14k/training",
+    "wikir/fr14k/validation",
+    "wikir/it16k/test",
+    "wikir/it16k/training",
+    "wikir/it16k/validation"
+]

requirements.txt

@@ -7,4 +7,5 @@ pyserini==0.21.0
 torch==2.0.1
 plotly==5.15.0
 captum==0.6.0
-protobuf==4.21.11
+protobuf==3.20.0
+beautifulsoup4==4.12.2

scripts/collect_ir_dataset_names.py

@@ -0,0 +1,26 @@
+import requests
+from bs4 import BeautifulSoup
+import re
+import json
+import os
+import pathlib
+import shutil
+
+
+
+def get_ir_dataset_names():
+    url = "https://raw.githubusercontent.com/allenai/ir_datasets/master/ir_datasets/etc/metadata.json"
+    # read in the json
+    with requests.get(url) as r:
+        data = json.loads(r.text)
+    names = []
+    for dataset in data:
+        if "docs" in data[dataset] and "queries" in data[dataset] and "qrels" in data[dataset]:
+            names.append(dataset)
+    return names
+
+
+if __name__ == "__main__":
+    names = get_ir_dataset_names()
+    with open("ir_dataset_names.json", "w") as fout:
+        json.dump(names, fout, indent=4)