claude test

app.py

@@ -23,7 +23,15 @@ from src.display.utils import (
     WeightType,
     Precision,
 )
-from src.envs import API, EVAL_RESULTS_PATH_CDM, EVAL_RESULTS_PATH_CDM_FI, REPO_ID, RESULTS_REPO_CDM, RESULTS_REPO_CDM_FI, TOKEN
+from src.envs import (
+    API,
+    EVAL_RESULTS_PATH_CDM,
+    EVAL_RESULTS_PATH_CDM_FI,
+    REPO_ID,
+    RESULTS_REPO_CDM,
+    RESULTS_REPO_CDM_FI,
+    TOKEN,
+)
 from src.populate import get_leaderboard_df
 
 
@@ -62,6 +70,7 @@ except Exception:
 LEADERBOARD_DF_CDM = get_leaderboard_df(EVAL_RESULTS_PATH_CDM, COLS, BENCHMARK_COLS)
 LEADERBOARD_DF_CDM_FI = get_leaderboard_df(EVAL_RESULTS_PATH_CDM_FI, COLS, BENCHMARK_COLS)
 
+
 def init_leaderboard(dataframe):
     if dataframe is None or dataframe.empty:
         raise ValueError("Leaderboard DataFrame is empty or None.")
@@ -74,18 +83,6 @@ def init_leaderboard(dataframe):
             label="Select Columns to Display:",
         ),
         search_columns=[AutoEvalColumn.model.name],
-        # hide_columns=[c.name for c in fields(AutoEvalColumn) if c.hidden],
-        # filter_columns=[
-            # ColumnFilter(AutoEvalColumn.model_type.name, type="checkboxgroup", label="Model types"),
-            # ColumnFilter(AutoEvalColumn.precision.name, type="checkboxgroup", label="Precision"),
-            # ColumnFilter(
-            #     AutoEvalColumn.seq_length.name,
-            #     type="checkboxgroup",
-            #     label="Sequence Lengths",
-            # )
-            # ColumnFilter(AutoEvalColumn.still_on_hub.name, type="boolean", label="Deleted/incomplete", default=True),
-        # ],
-        # bool_checkboxgroup_label="Hide models",
         interactive=False,
     )
 
@@ -97,10 +94,10 @@ with demo:
 
     with gr.Tabs(elem_classes="tab-buttons") as tabs:
         with gr.TabItem("MIMIC CDM", elem_id="llm-benchmark-tab-table", id=0):
-            leaderboard = init_leaderboard(LEADERBOARD_DF_CDM)
+            leaderboard_cdm = init_leaderboard(LEADERBOARD_DF_CDM)
 
         with gr.TabItem("MIMIC CDM FI", elem_id="llm-benchmark-tab-table", id=1):
-            leaderboard = init_leaderboard(LEADERBOARD_DF_CDM_FI)
+            leaderboard_cdm_fi = init_leaderboard(LEADERBOARD_DF_CDM_FI)
 
         with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):
             gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")

src/display/utils.py

@@ -5,6 +5,7 @@ import pandas as pd
 
 from src.about import Tasks
 
+
 def fields(raw_class):
     return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
 
@@ -20,15 +21,16 @@ class ColumnContent:
     hidden: bool = False
     never_hidden: bool = False
 
+
 ## Leaderboard columns
 auto_eval_column_dict = []
 # Init
 # auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
 auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
-#Scores
+# Scores
 auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
 for task in Tasks:
-    auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
+    auto_eval_column_dict.append([task.value.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
 # Model information
 # auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
 auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
@@ -37,7 +39,9 @@ auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Arch
 # auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub License", "str", False)])
 auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)])
 auto_eval_column_dict.append(["seq_length", ColumnContent, ColumnContent("Max Sequence Length", "number", False)])
-auto_eval_column_dict.append(["model_quantization_bits", ColumnContent, ColumnContent("Quantization Bits", "number", False)])
+auto_eval_column_dict.append(
+    ["model_quantization_bits", ColumnContent, ColumnContent("Quantization Bits", "number", False)]
+)
 # auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])
 auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])
 # auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
@@ -45,6 +49,7 @@ auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Avai
 # We use make dataclass to dynamically fill the scores from Tasks
 AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
 
+
 ## For the queue columns in the submission tab
 @dataclass(frozen=True)
 class EvalQueueColumn:  # Queue column
@@ -55,12 +60,13 @@ class EvalQueueColumn:  # Queue column
     weight_type = ColumnContent("weight_type", "str", "Original")
     status = ColumnContent("status", "str", True)
 
+
 ## All the model information that we might need
 @dataclass
 class ModelDetails:
     name: str
     display_name: str = ""
-    symbol: str = "" # emoji
+    symbol: str = ""  # emoji
 
 
 class ModelType(Enum):
@@ -85,18 +91,20 @@ class ModelType(Enum):
             return ModelType.IFT
         return ModelType.Unknown
 
+
 class WeightType(Enum):
     Adapter = ModelDetails("Adapter")
     Original = ModelDetails("Original")
     Delta = ModelDetails("Delta")
 
+
 class Precision(Enum):
     float16 = ModelDetails("float16")
     bfloat16 = ModelDetails("bfloat16")
     float32 = ModelDetails("float32")
-    #qt_8bit = ModelDetails("8bit")
-    #qt_4bit = ModelDetails("4bit")
-    #qt_GPTQ = ModelDetails("GPTQ")
+    # qt_8bit = ModelDetails("8bit")
+    # qt_4bit = ModelDetails("4bit")
+    # qt_GPTQ = ModelDetails("GPTQ")
     Unknown = ModelDetails("?")
 
     def from_str(precision):
@@ -106,14 +114,15 @@ class Precision(Enum):
             return Precision.bfloat16
         if precision in ["float32"]:
             return Precision.float32
-        #if precision in ["8bit"]:
+        # if precision in ["8bit"]:
         #    return Precision.qt_8bit
-        #if precision in ["4bit"]:
+        # if precision in ["4bit"]:
         #    return Precision.qt_4bit
-        #if precision in ["GPTQ", "None"]:
+        # if precision in ["GPTQ", "None"]:
         #    return Precision.qt_GPTQ
         return Precision.Unknown
 
+
 # Column selection
 COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
 
@@ -121,4 +130,3 @@ EVAL_COLS = [c.name for c in fields(EvalQueueColumn)]
 EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)]
 
 BENCHMARK_COLS = [t.value.col_name for t in Tasks]
-

src/leaderboard/read_evals.py

@@ -13,28 +13,35 @@ from src.display.utils import AutoEvalColumn, ModelType, Tasks, Precision, Weigh
 from transformers import AutoConfig
 from transformers.models.auto.tokenization_auto import AutoTokenizer
 
-def is_model_on_hub(model_name: str, revision: str, token: str = None, trust_remote_code=False, test_tokenizer=False) -> tuple[bool, str]:
+
+def is_model_on_hub(
+    model_name: str, revision: str, token: str = None, trust_remote_code=False, test_tokenizer=False
+) -> tuple[bool, str]:
     """Checks if the model model_name is on the hub, and whether it (and its tokenizer) can be loaded with AutoClasses."""
     try:
-        config = AutoConfig.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
+        config = AutoConfig.from_pretrained(
+            model_name, revision=revision, trust_remote_code=trust_remote_code, token=token
+        )
         if test_tokenizer:
             try:
-                tk = AutoTokenizer.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
+                tk = AutoTokenizer.from_pretrained(
+                    model_name, revision=revision, trust_remote_code=trust_remote_code, token=token
+                )
             except ValueError as e:
+                return (False, f"uses a tokenizer which is not in a transformers release: {e}", None)
+            except Exception as e:
                 return (
                     False,
-                    f"uses a tokenizer which is not in a transformers release: {e}",
-                    None
+                    "'s tokenizer cannot be loaded. Is your tokenizer class in a stable transformers release, and correctly configured?",
+                    None,
                 )
-            except Exception as e:
-                return (False, "'s tokenizer cannot be loaded. Is your tokenizer class in a stable transformers release, and correctly configured?", None)
         return True, None, config
 
     except ValueError:
         return (
             False,
             "needs to be launched with `trust_remote_code=True`. For safety reason, we do not allow these models to be automatically submitted to the leaderboard.",
-            None
+            None,
         )
 
     except Exception as e:
@@ -116,7 +123,6 @@ class EvalResult:
         model_quantization_bits = config.get("model_quantization_bits", 0)
         # print(self.seq_length)
 
-
         return self(
             eval_name=result_key,
             full_model=full_model,
@@ -128,7 +134,7 @@ class EvalResult:
             still_on_hub=still_on_hub,
             architecture=architecture,
             seq_length=seq_length,
-            model_quantization_bits=model_quantization_bits
+            model_quantization_bits=model_quantization_bits,
         )
 
     def update_with_request_file(self, requests_path):
@@ -151,28 +157,24 @@ class EvalResult:
 
     def to_dict(self):
         """Converts the Eval Result to a dict compatible with our dataframe display"""
-        # print(self.seq_length)
         average = sum([v for v in self.results.values() if v is not None]) / len(Tasks)
         data_dict = {
-            "eval_name": self.eval_name,  # not a column, just a save name,
-            # AutoEvalColumn.precision.name: self.precision.value.name,
-            # AutoEvalColumn.model_type.name: self.model_type.value.name,
-            # AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol,
-            # AutoEvalColumn.weight_type.name: self.weight_type.value.name,
+            "eval_name": self.eval_name,  # not a column, just a save name
             AutoEvalColumn.architecture.name: self.architecture,
             AutoEvalColumn.model.name: make_clickable_model(self.full_model),
-            # AutoEvalColumn.revision.name: self.revision,
-            AutoEvalColumn.average.name: average,
-            # AutoEvalColumn.license.name: self.license,
-            # AutoEvalColumn.likes.name: self.likes,
+            AutoEvalColumn.average.name: round(average, 2),  # Round to 2 decimal places
             AutoEvalColumn.params.name: self.params,
             AutoEvalColumn.seq_length.name: self.seq_length,
             AutoEvalColumn.model_quantization_bits.name: self.model_quantization_bits,
             AutoEvalColumn.still_on_hub.name: self.still_on_hub,
         }
 
+        # Add task results
         for task in Tasks:
-            data_dict[task.value.col_name] = self.results[task.value.benchmark]
+            if task.value.benchmark in self.results:
+                data_dict[task.value.col_name] = round(self.results[task.value.benchmark], 2)
+            else:
+                data_dict[task.value.col_name] = None
 
         return data_dict