fix bugs - 0819

1) no hyperlink for "BM25" -> add hyperlink
2) unique key of df -> set to timestamp

app.py

@@ -8,15 +8,57 @@ from src.about import (
     TITLE,
     EVALUATION_QUEUE_TEXT
 )
-from src.benchmarks import DOMAIN_COLS_QA, LANG_COLS_QA, DOMAIN_COLS_LONG_DOC, LANG_COLS_LONG_DOC, METRIC_LIST, \
-    DEFAULT_METRIC_QA, DEFAULT_METRIC_LONG_DOC
+from src.benchmarks import (
+    DOMAIN_COLS_QA,
+    LANG_COLS_QA,
+    DOMAIN_COLS_LONG_DOC,
+    LANG_COLS_LONG_DOC,
+    METRIC_LIST,
+    DEFAULT_METRIC_QA,
+    DEFAULT_METRIC_LONG_DOC
+)
 from src.display.css_html_js import custom_css
-from src.display.utils import COL_NAME_IS_ANONYMOUS, COL_NAME_REVISION, COL_NAME_TIMESTAMP, COL_NAME_RERANKING_MODEL, COL_NAME_RETRIEVAL_MODEL
-from src.envs import API, EVAL_RESULTS_PATH, REPO_ID, RESULTS_REPO, TOKEN
-from src.read_evals import get_raw_eval_results, get_leaderboard_df
-from src.utils import update_metric, upload_file, get_default_cols, submit_results, reset_rank, remove_html
-from src.display.gradio_formatting import get_version_dropdown, get_search_bar, get_reranking_dropdown, \
-    get_metric_dropdown, get_domain_dropdown, get_language_dropdown, get_anonymous_checkbox, get_revision_and_ts_checkbox, get_leaderboard_table, get_noreranking_dropdown
+from src.display.utils import (
+    COL_NAME_IS_ANONYMOUS,
+    COL_NAME_REVISION,
+    COL_NAME_TIMESTAMP,
+    COL_NAME_RERANKING_MODEL,
+    COL_NAME_RETRIEVAL_MODEL
+)
+from src.envs import (
+    API,
+    EVAL_RESULTS_PATH,
+    REPO_ID,
+    RESULTS_REPO,
+    TOKEN,
+    BM25_LINK,
+    BENCHMARK_VERSION_LIST,
+    LATEST_BENCHMARK_VERSION
+)
+from src.read_evals import (
+    get_raw_eval_results,
+    get_leaderboard_df
+)
+from src.utils import (
+    update_metric,
+    upload_file,
+    get_default_cols,
+    submit_results,
+    reset_rank,
+    remove_html
+)
+from src.display.gradio_formatting import (
+    get_version_dropdown,
+    get_search_bar,
+    get_reranking_dropdown,
+    get_metric_dropdown,
+    get_domain_dropdown,
+    get_language_dropdown,
+    get_anonymous_checkbox,
+    get_revision_and_ts_checkbox,
+    get_leaderboard_table,
+    get_noreranking_dropdown
+)
 from src.display.gradio_listener import set_listeners
 
 def restart_space():
@@ -32,7 +74,7 @@ except Exception as e:
     print(f'failed to download')
     restart_space()
 
-raw_data = get_raw_eval_results(f"{EVAL_RESULTS_PATH}/AIR-Bench_24.04")
+raw_data = get_raw_eval_results(f"{EVAL_RESULTS_PATH}/{LATEST_BENCHMARK_VERSION}")
 
 original_df_qa = get_leaderboard_df(
     raw_data, task='qa', metric=DEFAULT_METRIC_QA)
@@ -190,7 +232,7 @@ with demo:
                         queue=True
                     )
                 with gr.TabItem("Reranking Only", id=12):
-                    lb_df_reranker = leaderboard_df_qa[leaderboard_df_qa[COL_NAME_RETRIEVAL_MODEL] == "BM25"]
+                    lb_df_reranker = leaderboard_df_qa[leaderboard_df_qa[COL_NAME_RETRIEVAL_MODEL] == BM25_LINK]
                     lb_df_reranker = reset_rank(lb_df_reranker)
                     reranking_models_reranker = lb_df_reranker[COL_NAME_RERANKING_MODEL].apply(remove_html).unique().tolist()
                     with gr.Row():
@@ -199,7 +241,7 @@ with demo:
                         with gr.Column(scale=1):
                             search_bar_reranker = gr.Textbox(show_label=False, visible=False)
                     lb_table_reranker = get_leaderboard_table(lb_df_reranker, types_qa)
-                    hidden_lb_df_reranker = original_df_qa[original_df_qa[COL_NAME_RETRIEVAL_MODEL] == "BM25"]
+                    hidden_lb_df_reranker = original_df_qa[original_df_qa[COL_NAME_RETRIEVAL_MODEL] == BM25_LINK]
                     hidden_lb_df_reranker = reset_rank(hidden_lb_df_reranker)
                     hidden_lb_table_reranker = get_leaderboard_table(
                         hidden_lb_df_reranker, types_qa, visible=False
@@ -345,7 +387,7 @@ with demo:
                     )
                 with gr.TabItem("Reranking Only", id=22):
                     lb_df_reranker_ldoc = leaderboard_df_long_doc[
-                        leaderboard_df_long_doc[COL_NAME_RETRIEVAL_MODEL] == "BM25"
+                        leaderboard_df_long_doc[COL_NAME_RETRIEVAL_MODEL] == BM25_LINK
                         ]
                     lb_df_reranker_ldoc = reset_rank(lb_df_reranker_ldoc)
                     reranking_models_reranker_ldoc = lb_df_reranker_ldoc[COL_NAME_RERANKING_MODEL].apply(remove_html).unique().tolist()
@@ -355,7 +397,7 @@ with demo:
                         with gr.Column(scale=1):
                             search_bar_reranker_ldoc = gr.Textbox(show_label=False, visible=False)
                     lb_table_reranker_ldoc = get_leaderboard_table(lb_df_reranker_ldoc, types_long_doc)
-                    hidden_lb_df_reranker_ldoc = original_df_long_doc[original_df_long_doc[COL_NAME_RETRIEVAL_MODEL] == "BM25"]
+                    hidden_lb_df_reranker_ldoc = original_df_long_doc[original_df_long_doc[COL_NAME_RETRIEVAL_MODEL] == BM25_LINK]
                     hidden_lb_df_reranker_ldoc = reset_rank(hidden_lb_df_reranker_ldoc)
                     hidden_lb_table_reranker_ldoc = get_leaderboard_table(
                         hidden_lb_df_reranker_ldoc, types_long_doc, visible=False
@@ -414,8 +456,8 @@ with demo:
                 with gr.Row():
                     with gr.Column():
                         benchmark_version = gr.Dropdown(
-                            ["AIR-Bench_24.04", ],
-                            value="AIR-Bench_24.04",
+                            BENCHMARK_VERSION_LIST,
+                            value=LATEST_BENCHMARK_VERSION,
                             interactive=True,
                             label="AIR-Bench Version")
                 with gr.Row():

src/display/formatting.py

@@ -4,7 +4,7 @@ def model_hyperlink(link, model_name):
 
 def make_clickable_model(model_name: str, model_link: str):
     # link = f"https://huggingface.co/{model_name}"
-    if not model_link or not model_link.startswith("https://") or model_name == "BM25":
+    if not model_link or not model_link.startswith("https://"):
         return model_name
     return model_hyperlink(model_link, model_name)
 

src/display/gradio_formatting.py

@@ -1,10 +1,10 @@
 import gradio as gr
-
+from src.envs import BENCHMARK_VERSION_LIST, LATEST_BENCHMARK_VERSION
 
 def get_version_dropdown():
     return gr.Dropdown(
-        choices=["AIR-Bench_24.04", ],
-        value="AIR-Bench_24.04",
+        choices=BENCHMARK_VERSION_LIST,
+        value=LATEST_BENCHMARK_VERSION,
         label="Select the version of AIR-Bench",
         interactive=True
     )

src/envs.py

@@ -1,5 +1,5 @@
 import os
-
+from display.formatting import model_hyperlink
 from huggingface_hub import HfApi
 
 # Info to change for your repository
@@ -22,3 +22,12 @@ CACHE_PATH = os.getenv("HF_HOME", ".")
 EVAL_RESULTS_PATH = os.path.join(CACHE_PATH, "eval_results")
 
 API = HfApi(token=TOKEN)
+
+BM25_LINK = model_hyperlink("https://github.com/castorini/pyserini", "BM25")
+
+BENCHMARK_VERSION_LIST = [
+    "AIR-Bench_24.04",
+    # "AIR-Bench_24.05",
+]
+
+LATEST_BENCHMARK_VERSION = BENCHMARK_VERSION_LIST[-1]

src/read_evals.py

@@ -174,8 +174,8 @@ def get_raw_eval_results(results_path: str) -> List[FullEvalResult]:
             print(f"loading file failed. {model_result_filepath}")
             continue
         print(f'file loaded: {model_result_filepath}')
-        eval_name = eval_result.eval_name
-        eval_results[eval_name] = eval_result
+        timestamp = eval_result.timestamp
+        eval_results[timestamp] = eval_result
 
     results = []
     for k, v in eval_results.items():

src/utils.py

@@ -10,7 +10,7 @@ from src.benchmarks import BENCHMARK_COLS_QA, BENCHMARK_COLS_LONG_DOC, Benchmark
 from src.display.formatting import styled_message, styled_error
 from src.display.utils import COLS_QA, TYPES_QA, COLS_LONG_DOC, TYPES_LONG_DOC, COL_NAME_RANK, COL_NAME_AVG, \
     COL_NAME_RERANKING_MODEL, COL_NAME_RETRIEVAL_MODEL, COL_NAME_IS_ANONYMOUS, COL_NAME_TIMESTAMP, COL_NAME_REVISION, get_default_auto_eval_column_dict
-from src.envs import API, SEARCH_RESULTS_REPO
+from src.envs import API, SEARCH_RESULTS_REPO, LATEST_BENCHMARK_VERSION
 from src.read_evals import FullEvalResult, get_leaderboard_df, calculate_mean
 
 import re
@@ -251,7 +251,7 @@ def submit_results(
         model_url: str,
         reranking_model: str="",
         reranking_model_url: str="",
-        version: str="AIR-Bench_24.04",
+        version: str=LATEST_BENCHMARK_VERSION,
         is_anonymous=False):
     if not filepath.endswith(".zip"):
         return styled_error(f"file uploading aborted. wrong file type: {filepath}")