add

app.py

@@ -50,7 +50,7 @@ def restart_space():
 #     restart_space()
 
 
-LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)
+LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS, task="Overall")
 
 (
     finished_eval_queue_df,
@@ -111,10 +111,15 @@ with demo:
 
     with gr.Tabs(elem_classes="tab-buttons") as tabs:
         with gr.TabItem("🏅 LLM Benchmark", elem_id="llm-benchmark-tab-table", id=0):
-            leaderboard = init_leaderboard(LEADERBOARD_DF)
+            # leaderboard = init_leaderboard(LEADERBOARD_DF)
+            with gr.Tabs():
+                with gr.TabItem("Overall", elem_id="overall", id=0):
+                    pdb.set_trace()
+                    leaderboard = init_leaderboard(LEADERBOARD_DF)
 
-        # with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):
-        #     gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
+
+        with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):
+            gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
 
         # with gr.TabItem("🚀 Submit here! ", elem_id="llm-benchmark-tab-table", id=3):
         #     with gr.Column():

src/about.py

@@ -19,6 +19,19 @@ class Tasks(Enum):
     task4 = Task("sudoku", "EM", "Sudoku")
     task5 = Task("drop_quote", "EM", "Drop Quote")
 
+@dataclass
+class Metric:
+    short: str
+    col_name: str
+
+class Metrics(Enum):
+    CR = Metric("CR", "Completion Rate")
+    S_Acc = Metric("S-Acc", "Subtask Accuracy")
+    EM = Metric("EM", "Exact Match")
+    PM_05 = Metric("PM-0.5", "Partial Match (0.5)")
+    Tokens = Metric("Tokens", "Tokens")
+
+
 NUM_FEWSHOT = 0 # Change with your few shot
 # ---------------------------------------------------
 

src/display/css_html_js.py

@@ -1,22 +1,4 @@
 custom_css = """
-/* 修改排序按钮颜色、大小等 */
-.gr-datatable .sorting:before, 
-.gr-datatable .sorting:after {
-    color: #007bff !important; /* 修改排序图标颜色 */
-    font-size: 16px !important; /* 调整大小 */
-}
-
-/* 鼠标悬停时改变颜色 */
-.gr-datatable .sorting:hover:before, 
-.gr-datatable .sorting:hover:after {
-    color: #ff4500 !important; /* 悬停时变色 */
-}
-
-/* 激活的排序图标 */
-.gr-datatable .sorting_asc:before,
-.gr-datatable .sorting_desc:before {
-    color: #28a745 !important; /* 绿色代表当前排序状态 */
-}
 
 .markdown-text {
     font-size: 16px !important;

src/display/utils.py

@@ -3,7 +3,7 @@ from enum import Enum
 
 import pandas as pd
 
-from src.about import Tasks
+from src.about import Tasks, Metrics
 
 def fields(raw_class):
     return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
@@ -25,13 +25,15 @@ auto_eval_column_dict = []
 # Init
 # auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
 auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
+auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", True)])
 # auto_eval_column_dict.append(["model", ColumnContent, field(default_factory=lambda: ColumnContent("Model", "markdown", True, never_hidden=True))])
 #Scores
-auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("EM ⬆️", "number", True)])
+# auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("EM ⬆️", "number", True)])
 # auto_eval_column_dict.append(["average", ColumnContent, field(default_factory=lambda: ColumnContent("Average ⬆️", "number", True))])
-for task in Tasks:
-    auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
-    # auto_eval_column_dict.append([task.name, ColumnContent, field(default_factory=lambda task=task: ColumnContent(task.value.col_name, "number", True))])
+# for task in Tasks:
+#     auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
+for metric in Metrics:
+    auto_eval_column_dict.append([metric.name, ColumnContent, ColumnContent(metric.value.col_name, "number", True)])
 # Model information
 # auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
 # auto_eval_column_dict.append(["model_type", ColumnContent, field(default_factory=lambda: ColumnContent("Type", "str", False))])
@@ -39,7 +41,7 @@ for task in Tasks:
 # auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])
 # auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])
 # auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub License", "str", False)])
-auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)])
+# auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)])
 # auto_eval_column_dict.append(["params", ColumnContent, field(default_factory=lambda: ColumnContent("#Params (B)", "number", False))])
 # auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])
 # auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])

src/leaderboard/read_evals.py

@@ -8,7 +8,7 @@ import dateutil
 import numpy as np
 
 from src.display.formatting import make_clickable_model
-from src.display.utils import AutoEvalColumn, ModelType, Tasks, Precision, WeightType
+from src.display.utils import AutoEvalColumn, ModelType, Tasks, Precision, WeightType, Metrics
 from src.submission.check_validity import is_model_on_hub
 
 
@@ -34,7 +34,7 @@ class EvalResult:
     link: str = ''
 
     @classmethod
-    def init_from_json_file(self, json_filepath):
+    def init_from_json_file(self, json_filepath, task):
         """Inits the result from the specific model result file"""
         with open(json_filepath) as fp:
             data = json.load(fp)
@@ -73,16 +73,21 @@ class EvalResult:
 
         # Extract results available in this file (some results are split in several files)
         results = {}
-        for task in Tasks:
-            task = task.value
+        # for task in Tasks:
+        #     task = task.value
 
-            # We average all scores of a given metric (not all metrics are present in all files)
-            accs = np.array([float(v.get(task.metric, None)) for k, v in data["results"].items() if task.benchmark == k.lower()])
-            if accs.size == 0 or any([acc is None for acc in accs]):
-                continue
+        #     # We average all scores of a given metric (not all metrics are present in all files)
+        #     accs = np.array([float(v.get(task.metric, None)) for k, v in data["results"].items() if task.benchmark == k.lower()])
+        #     if accs.size == 0 or any([acc is None for acc in accs]):
+        #         continue
 
-            mean_acc = np.mean(accs)
-            results[task.benchmark] = mean_acc
+        #     mean_acc = np.mean(accs)
+        #     results[task.benchmark] = mean_acc
+        # import pdb; pdb.set_trace()
+        for metric in Metrics:
+            metric = metric.value
+
+            results[metric.short] = data["results"][task][metric.short]
 
         return self(
             eval_name=result_key,
@@ -118,7 +123,7 @@ class EvalResult:
 
     def to_dict(self):
         """Converts the Eval Result to a dict compatible with our dataframe display"""
-        average = sum([v for v in self.results.values() if v is not None]) / len(Tasks)
+        # average = sum([v for v in self.results.values() if v is not None]) / len(Tasks)
         data_dict = {
             "eval_name": self.eval_name,  # not a column, just a save name,
             # AutoEvalColumn.precision.name: self.precision.value.name,
@@ -128,15 +133,17 @@ class EvalResult:
             # AutoEvalColumn.architecture.name: self.architecture,
             AutoEvalColumn.model.name: make_clickable_model(self.full_model, self.link),
             # AutoEvalColumn.revision.name: self.revision,
-            AutoEvalColumn.average.name: average,
+            # AutoEvalColumn.average.name: average,
             # AutoEvalColumn.license.name: self.license,
             # AutoEvalColumn.likes.name: self.likes,
             AutoEvalColumn.params.name: self.num_params,
             # AutoEvalColumn.still_on_hub.name: self.still_on_hub,
         }
 
-        for task in Tasks:
-            data_dict[task.value.col_name] = self.results[task.value.benchmark]
+        # for task in Tasks:
+        #     data_dict[task.value.col_name] = self.results[task.value.benchmark]
+        for metric in Metrics:
+            data_dict[metric.value.col_name] = self.results[metric.value.short]
 
         return data_dict
 
@@ -164,7 +171,7 @@ class EvalResult:
 #     return request_file
 
 
-def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResult]:
+def get_raw_eval_results(results_path: str, requests_path: str, task: str) -> list[EvalResult]:
     """From the path of the results folder root, extract all needed info for results"""
     model_result_filepaths = []
 
@@ -185,7 +192,7 @@ def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResu
     eval_results = {}
     for model_result_filepath in model_result_filepaths:
         # Creation of result
-        eval_result = EvalResult.init_from_json_file(model_result_filepath)
+        eval_result = EvalResult.init_from_json_file(model_result_filepath, task)
         # eval_result.update_with_request_file(requests_path)
 
         # Store results of same eval together

src/populate.py

@@ -8,17 +8,17 @@ from src.display.utils import AutoEvalColumn, EvalQueueColumn
 from src.leaderboard.read_evals import get_raw_eval_results
 
 
-def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
+def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchmark_cols: list, task) -> pd.DataFrame:
     """Creates a dataframe from all the individual experiment results"""
-    raw_data = get_raw_eval_results(results_path, requests_path)
+    raw_data = get_raw_eval_results(results_path, requests_path, task)
     all_data_json = [v.to_dict() for v in raw_data]
 
     df = pd.DataFrame.from_records(all_data_json)
-    df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
+    df = df.sort_values(by=["Exact Match"], ascending=False)
     df = df[cols].round(decimals=2)
 
     # filter out if any of the benchmarks have not been produced
-    df = df[has_no_nan_values(df, benchmark_cols)]
+    # df = df[has_no_nan_values(df, benchmark_cols)]
     return df