update

app.py

@@ -5,7 +5,7 @@ from apscheduler.schedulers.background import BackgroundScheduler
 from huggingface_hub import snapshot_download
 import plotly.graph_objects as go
 import plotly.express as px
-from src.about import Tasks, AssetTasks
+from src.about import Tasks, AssetTasks, UncertaintyTasks
 
 from src.about import (
     CITATION_BUTTON_LABEL,
@@ -21,10 +21,12 @@ from src.display.utils import (
     ASSET_BENCHMARK_COLS,
     COLS,
     ASSET_COLS,
+    UNCERTAINTY_COLS,
     EVAL_COLS,
     EVAL_TYPES,
     AutoEvalColumn,
     AutoEvalColumnAsset,
+    AutoEvalColumnUncertainty,
     ModelType,
     fields,
     WeightType,
@@ -63,6 +65,8 @@ print(ASSET_COLS)
 
 ASSET_LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, ASSET_COLS, ASSET_BENCHMARK_COLS, AssetTasks)
 
+UNCERTAINTY_LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, UNCERTAINTY_COLS, ASSET_BENCHMARK_COLS, UncertaintyTasks)
+
 
 (
     finished_eval_queue_df,
@@ -165,8 +169,11 @@ with demo:
         
         with gr.TabItem("🛠️ Asset Benchmark", elem_id="llm-benchmark-asset-tab-table", id=1):
             leaderboard = init_leaderboard(ASSET_LEADERBOARD_DF, AutoEvalColumnAsset)
+        
+        with gr.TabItem("😵‍💫 Uncertainty Benchmark", elem_id="llm-benchmark-asset-tab-table", id=2):
+            leaderboard = init_leaderboard(UNCERTAINTY_LEADERBOARD_DF, AutoEvalColumnUncertainty)
 
-        with gr.TabItem("📊 Performance Plot", elem_id="llm-benchmark-tab-table", id=2):
+        with gr.TabItem("📊 Performance Plot", elem_id="llm-benchmark-tab-table", id=3):
             print(LEADERBOARD_DF.columns)
             # gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
             perf_plot = gr.components.Plot(

src/about.py

@@ -17,7 +17,6 @@ class Tasks(Enum):
     task2 = Task("acc_el", "acc_el", "Acc_El")
     task3 = Task("acc_perturb", "perturb_score", "Acc_Perturb")
     task4 = Task("score_consistency", "consist_score", "Consistency_Score")
-    task5 = Task("uncertainty", "uncertainty_score", "Uncertainty_Score")
     
 class AssetTasks(Enum):
     # task_key in the json file, metric_key in the json file, name to display in the leaderboard 
@@ -32,6 +31,11 @@ class AssetTasks(Enum):
     task8 = Task("acc_fan", "acc_fan", "acc_fan")
     task9 = Task("acc_power_transformer", "acc_power_transformer", "acc_power_transformer")
 
+class UncertaintyTasks(Enum):
+    # task_key in the json file, metric_key in the json file, name to display in the leaderboard 
+    task0 = Task("fmsr_ss", "fmsr_ss", "fmsr_ss")
+    task1 = Task("fmsr_coverage_rate", "fmsr_coverage_rate", "fmsr_coverage_rate")
+    task2 = Task("fmsr_uacc", "fmsr_uacc", "fmsr_uacc")
 
 # {
 #   "acc_overall": {

src/display/utils.py

@@ -3,7 +3,7 @@ from enum import Enum
 
 import pandas as pd
 
-from src.about import Tasks, AssetTasks
+from src.about import Tasks, AssetTasks, UncertaintyTasks
 
 def fields(raw_class):
     return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
@@ -50,6 +50,9 @@ auto_eval_column_asset_dict = get_auto_eval_column_dict(AssetTasks)
 # We use make dataclass to dynamically fill the scores from Tasks
 AutoEvalColumnAsset = make_dataclass("AutoEvalColumnAsset", auto_eval_column_asset_dict, frozen=True)
 
+auto_eval_column_uncertainty_dict = get_auto_eval_column_dict(UncertaintyTasks)
+# We use make dataclass to dynamically fill the scores from Tasks
+AutoEvalColumnUncertainty = make_dataclass("AutoEvalColumnUncertainty", auto_eval_column_uncertainty_dict, frozen=True)
 
 ## For the queue columns in the submission tab
 @dataclass(frozen=True)
@@ -111,10 +114,13 @@ class Precision(Enum):
 # Column selection
 COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
 ASSET_COLS = [c.name for c in fields(AutoEvalColumnAsset) if not c.hidden]
+UNCERTAINTY_COLS = [c.name for c in fields(AutoEvalColumnUncertainty) if not c.hidden]
+
 
 EVAL_COLS = [c.name for c in fields(EvalQueueColumn)]
 EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)]
 
 BENCHMARK_COLS = [t.value.col_name for t in Tasks]
 ASSET_BENCHMARK_COLS = [t.value.col_name for t in AssetTasks]
+ASSET_BENCHMARK_COLS = [t.value.col_name for t in UncertaintyTasks]
 

src/leaderboard/read_evals.py

@@ -114,7 +114,7 @@ class EvalResult:
     def to_dict(self, task_class):
         """Converts the Eval Result to a dict compatible with our dataframe display"""
         #ignore uncertainty for overall calculation
-        scores = [v for k, v in zip(self.results.keys(), self.results.values()) if v is not None and k != 'uncertainty']
+        scores = [v for k, v in zip(self.results.keys(), self.results.values()) if v is not None]
         average = sum(scores) / len(scores)
         # average = sum([v for v in self.results.values() if v is not None]) / len(task_class)
         data_dict = {