debug config

app.py

@@ -78,13 +78,13 @@ def init_leaderboard(dataframe):
         filter_columns=[
             # ColumnFilter(AutoEvalColumn.model_type.name, type="checkboxgroup", label="Model types"),
             # ColumnFilter(AutoEvalColumn.precision.name, type="checkboxgroup", label="Precision"),
-            ColumnFilter(
-                AutoEvalColumn.params.name,
-                type="slider",
-                min=0.01,
-                max=150,
-                label="Select the number of parameters (B)",
-            ),
+            # ColumnFilter(
+            #     AutoEvalColumn.params.name,
+            #     type="slider",
+            #     min=0.01,
+            #     max=150,
+            #     label="Select the number of parameters (B)",
+            # ),
             ColumnFilter(AutoEvalColumn.still_on_hub.name, type="boolean", label="Deleted/incomplete", default=True),
         ],
         bool_checkboxgroup_label="Hide models",

src/about.py

@@ -59,36 +59,6 @@ python run_full_info.py pathology=diverticulitis model=<YOUR_MODEL_NAME>
 
 """
 
-# EVALUATION_QUEUE_TEXT = """
-# ## Some good practices before submitting a model
-
-# ### 1) Make sure you can load your model and tokenizer using AutoClasses:
-# ```python
-# from transformers import AutoConfig, AutoModel, AutoTokenizer
-# config = AutoConfig.from_pretrained("your model name", revision=revision)
-# model = AutoModel.from_pretrained("your model name", revision=revision)
-# tokenizer = AutoTokenizer.from_pretrained("your model name", revision=revision)
-# ```
-# If this step fails, follow the error messages to debug your model before submitting it. It's likely your model has been improperly uploaded.
-
-# Note: make sure your model is public!
-# Note: if your model needs `use_remote_code=True`, we do not support this option yet but we are working on adding it, stay posted!
-
-# ### 2) Convert your model weights to [safetensors](https://huggingface.co/docs/safetensors/index)
-# It's a new format for storing weights which is safer and faster to load and use. It will also allow us to add the number of parameters of your model to the `Extended Viewer`!
-
-# ### 3) Make sure your model has an open license!
-# This is a leaderboard for Open LLMs, and we'd love for as many people as possible to know they can use your model 🤗
-
-# ### 4) Fill up your model card
-# When we add extra information about models to the leaderboard, it will be automatically taken from the model card
-
-# ## In case of model failure
-# If your model is displayed in the `FAILED` category, its execution stopped.
-# Make sure you have followed the above steps first.
-# If everything is done, check you can launch the EleutherAIHarness on your model locally, using the above command without modifications (you can add `--limit` to limit the number of examples per task).
-# """
-
 CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
 CITATION_BUTTON_TEXT = r"""
 @article{hager_evaluation_2024,

src/envs.py

@@ -12,7 +12,6 @@ OWNER = (
 # ----------------------------------
 
 REPO_ID = f"{OWNER}/leaderboard"
-# QUEUE_REPO = f"{OWNER}/requests"
 RESULTS_REPO_CDM = f"{OWNER}/results-CDM"
 RESULTS_REPO_CDM_FI = f"{OWNER}/results-CDM-FI"
 
@@ -20,10 +19,8 @@ RESULTS_REPO_CDM_FI = f"{OWNER}/results-CDM-FI"
 CACHE_PATH = os.getenv("HF_HOME", ".")
 
 # Local caches
-# EVAL_REQUESTS_PATH = os.path.join(CACHE_PATH, "eval-queue")
 EVAL_RESULTS_PATH_CDM = os.path.join(CACHE_PATH, "eval-results-CDM")
 EVAL_RESULTS_PATH_CDM_FI = os.path.join(CACHE_PATH, "eval-results-CDM-FI")
-# EVAL_REQUESTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-queue-bk")
 EVAL_RESULTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-results-bk")
 
 API = HfApi(token=TOKEN)

src/leaderboard/read_evals.py

@@ -41,6 +41,7 @@ class EvalResult:
             data = json.load(fp)
 
         config = data.get("config")
+        print(config)
 
         # Precision
         precision = Precision.from_str(config.get("model_dtype"))