Dramatically simplify everything

app.py

@@ -2,73 +2,37 @@ import gradio as gr
 import pandas as pd
 from apscheduler.schedulers.background import BackgroundScheduler
 from gradio_leaderboard import ColumnFilter, Leaderboard, SelectColumns
-from huggingface_hub import snapshot_download
 
+from leaderboard.populate import load_results
+from leaderboard.utils import COLUMNS
 from src.about import INTRODUCTION_TEXT, TITLE
 from src.display.css_html_js import custom_css
-from src.display.utils import BENCHMARK_COLS, COLS, EVAL_COLS, EVAL_TYPES, AutoEvalColumn, fields
-from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
-from src.populate import get_evaluation_queue_df, get_leaderboard_df
-from src.submission.submit import add_new_eval
+from src.envs import API, REPO_ID
 
 
 def restart_space():
     API.restart_space(repo_id=REPO_ID)
 
 
-### Space initialisation
-try:
-    print(EVAL_REQUESTS_PATH)
-    snapshot_download(
-        repo_id=QUEUE_REPO,
-        local_dir=EVAL_REQUESTS_PATH,
-        repo_type="dataset",
-        tqdm_class=None,
-        etag_timeout=30,
-        token=TOKEN,
-    )
-except Exception:
-    restart_space()
-try:
-    print(EVAL_RESULTS_PATH)
-    snapshot_download(
-        repo_id=RESULTS_REPO,
-        local_dir=EVAL_RESULTS_PATH,
-        repo_type="dataset",
-        tqdm_class=None,
-        etag_timeout=30,
-        token=TOKEN,
-    )
-except Exception:
-    restart_space()
-
-
-LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)
-
-(
-    finished_eval_queue_df,
-    running_eval_queue_df,
-    pending_eval_queue_df,
-) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
-
+def init_leaderboard(dataframe: pd.DataFrame):
+    dataframe = load_results()
 
-def init_leaderboard(dataframe):
     if dataframe is None or dataframe.empty:
         raise ValueError("Leaderboard DataFrame is empty or None.")
+
     return Leaderboard(
         value=dataframe,
-        datatype=[c.type for c in fields(AutoEvalColumn)],
+        datatype=[c.type for c in COLUMNS],
         select_columns=SelectColumns(
-            default_selection=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default],
-            cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden],
+            default_selection=[c.name for c in COLUMNS if c.displayed_by_default],
+            cant_deselect=[c.name for c in COLUMNS if c.never_hidden],
             label="Select Columns to Display:",
         ),
-        search_columns=[AutoEvalColumn.model_id.name],
-        hide_columns=[c.name for c in fields(AutoEvalColumn) if c.hidden],
+        search_columns=["Model"],
+        hide_columns=[c.name for c in COLUMNS if c.hidden],
         filter_columns=[
-            ColumnFilter(AutoEvalColumn.inference_provider.name, type="checkboxgroup", label="Inference Provider"),
             ColumnFilter(
-                AutoEvalColumn.average.name,
+                "Average",
                 type="slider",
                 min=0.01,
                 max=100,
@@ -87,79 +51,10 @@ with demo:
 
     with gr.Tabs(elem_classes="tab-buttons") as tabs:
         with gr.TabItem("🏅 LLM Benchmark", elem_id="llm-benchmark-tab-table", id=0):
-            leaderboard = init_leaderboard(LEADERBOARD_DF)
-
-        with gr.TabItem("🚀 Submit here! ", elem_id="llm-benchmark-tab-table", id=3):
-            with gr.Column():
-                with gr.Row():
-                    gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
-
-                with gr.Column():
-                    with gr.Accordion(
-                        f"✅ Finished Evaluations ({len(finished_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            finished_eval_table = gr.components.Dataframe(
-                                value=finished_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-                    with gr.Accordion(
-                        f"🔄 Running Evaluation Queue ({len(running_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            running_eval_table = gr.components.Dataframe(
-                                value=running_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-
-                    with gr.Accordion(
-                        f"⏳ Pending Evaluation Queue ({len(pending_eval_queue_df)})",
-                        open=False,
-                    ):
-                        with gr.Row():
-                            pending_eval_table = gr.components.Dataframe(
-                                value=pending_eval_queue_df,
-                                headers=EVAL_COLS,
-                                datatype=EVAL_TYPES,
-                                row_count=5,
-                            )
-            with gr.Row():
-                gr.Markdown("# ✉️✨ Submit your model here!", elem_classes="markdown-text")
-
-            with gr.Row():
-                with gr.Column():
-                    model_id_textbox = gr.Textbox(label="Model ID")
-                    inference_provider = gr.Dropdown(
-                        choices=[
-                            "Together",
-                            "OpenAI",
-                            "Anthropic",
-                            "Vertex AI",
-                        ],
-                        label="Inference Provider",
-                        multiselect=False,
-                        value=None,
-                        interactive=True,
-                    )
+            leaderboard = init_leaderboard()
 
-            submit_button = gr.Button("Submit Eval")
-            submission_result = gr.Markdown()
-            submit_button.click(
-                add_new_eval,
-                [
-                    model_id_textbox,
-                    inference_provider,
-                ],
-                submission_result,
-            )
 
 scheduler = BackgroundScheduler()
-scheduler.add_job(restart_space, "interval", seconds=1800)
+scheduler.add_job(restart_space, "interval", seconds=3600)
 scheduler.start()
 demo.queue(default_concurrency_limit=40).launch()

src/about.py

@@ -1,19 +1,3 @@
-from dataclasses import dataclass
-from enum import Enum
-
-
-@dataclass
-class Task:
-    benchmark: str
-    metric: str
-    col_name: str
-
-
-class Tasks(Enum):
-    # task_key in the json file, metric_key in the json file, name to display in the leaderboard
-    robloxQA = Task("robloxqa", "acc", "RobloxQA")
-
-
 # Your leaderboard name
 TITLE = """<h1 align="center" id="space-title">Roblox LLM Leaderboard</h1>"""
 

src/display/utils.py

@@ -1,62 +0,0 @@
-from dataclasses import dataclass, make_dataclass
-from enum import Enum
-
-import pandas as pd
-
-from src.about import Tasks
-
-
-def fields(raw_class):
-    return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
-
-
-# These classes are for user facing column names,
-# to avoid having to change them all around the code
-# when a modif is needed
-@dataclass
-class ColumnContent:
-    name: str
-    type: str
-    displayed_by_default: bool
-    hidden: bool = False
-    never_hidden: bool = False
-
-
-## For the queue columns in the submission tab
-@dataclass(frozen=True)
-class EvalQueueColumn:  # Queue column
-    model_id = ColumnContent("model_id", "str", True)
-    inference_provider = ColumnContent("inference_provider", "str", True)
-    status = ColumnContent("status", "str", True)
-    date = ColumnContent("date", "str", True)
-
-
-## All the model information that we might need
-@dataclass
-class ModelDetails:
-    name: str
-    display_name: str = ""
-    symbol: str = ""  # emoji
-
-
-## Leaderboard columns
-auto_eval_column_dict = [
-    ["model_id", ColumnContent, ColumnContent("Model", "str", True, never_hidden=True)],
-    ["inference_provider", ColumnContent, ColumnContent("Inference Provider", "str", False)],
-]
-# Scores
-auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
-for task in Tasks:
-    auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
-
-# We use make dataclass to dynamically fill the scores from Tasks
-AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
-
-
-# Column selection
-COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
-
-EVAL_COLS = [c.name for c in fields(EvalQueueColumn)]
-EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)]
-
-BENCHMARK_COLS = [t.value.col_name for t in Tasks]

src/envs.py

@@ -1,4 +1,5 @@
 import os
+from pathlib import Path
 
 from huggingface_hub import HfApi
 
@@ -7,19 +8,12 @@ from huggingface_hub import HfApi
 TOKEN = os.environ.get("HF_TOKEN")  # A read/write token for your org
 
 OWNER = "boatbomber"
-# ----------------------------------
-
 REPO_ID = f"{OWNER}/roblox-llm-leaderboard"
-QUEUE_REPO = f"{OWNER}/roblox-llm-leaderboard-requests"
 RESULTS_REPO = f"{OWNER}/roblox-llm-leaderboard-results"
 
 # If you setup a cache later, just change HF_HOME
-CACHE_PATH = os.getenv("HF_HOME", ".")
+CACHE_PATH = Path(os.getenv("HF_HOME", "."))
 
-# Local caches
-EVAL_REQUESTS_PATH = os.path.join(CACHE_PATH, "eval-queue")
-EVAL_RESULTS_PATH = os.path.join(CACHE_PATH, "eval-results")
-EVAL_REQUESTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-queue-bk")
-EVAL_RESULTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-results-bk")
+RESULTS_REPO_PATH = CACHE_PATH / "roblox-llm-leaderboard-results"
 
 API = HfApi(token=TOKEN)

src/leaderboard/populate.py

@@ -0,0 +1,47 @@
+import json
+
+import pandas as pd
+from huggingface_hub import snapshot_download
+
+from leaderboard.utils import COLUMNS
+from src.envs import RESULTS_REPO, RESULTS_REPO_PATH, TOKEN
+
+
+def download_result_data():
+    print(f"Downloading {RESULTS_REPO}")
+    snapshot_download(
+        repo_id=RESULTS_REPO,
+        local_dir=RESULTS_REPO_PATH,
+        repo_type="dataset",
+        tqdm_class=None,
+        etag_timeout=30,
+        token=TOKEN,
+    )
+
+
+def load_results() -> pd.DataFrame:
+    if not RESULTS_REPO_PATH.exists():
+        download_result_data()
+
+    data = []
+
+    results_dir = RESULTS_REPO_PATH / "results"
+    if not (results_dir.exists() and results_dir.is_dir()):
+        raise ValueError("No results found in the results directory")
+
+    for file in results_dir.rglob("*.json"):
+        with open(file) as f:
+            evaluation = json.load(f)
+
+            results = evaluation["Results"]
+            del evaluation["Results"]
+
+            evaluation["Average"] = sum(results.values()) / len(results)
+            for key, value in results.items():
+                evaluation[key] = value
+
+            data.append(evaluation)
+
+    dataframe = pd.DataFrame(data, columns=[c.name for c in COLUMNS])
+    dataframe.sort_values(by=["Average"], ascending=False)
+    return dataframe

src/leaderboard/read_evals.py

@@ -1,110 +0,0 @@
-import glob
-import json
-import math
-import os
-from dataclasses import dataclass, field
-
-import dateutil
-import numpy as np
-
-from src.display.utils import AutoEvalColumn, Tasks
-
-
-@dataclass
-class EvalResult:
-    """Represents one full evaluation."""
-
-    eval_name: str = ""
-    date: str = ""
-    model_id: str = ""
-    inference_provider: str = ""
-    results: dict = field(default_factory=dict)
-
-    @classmethod
-    def init_from_json_file(self, json_filepath):
-        """Inits the result from the specific model result file"""
-        with open(json_filepath) as fp:
-            data = json.load(fp)
-
-        raw_results = data.get("results", {})
-        config = data.get("config")
-        inference_provider = config.get("inference_provider", "Unknown")
-        model_id = config.get("model_id", "Unknown")
-        date = config.get("date", "Unknown")
-
-        # Extract results available in this file (some results are split in several files)
-        results = {}
-        for task in Tasks:
-            task = task.value
-
-            # We average all scores of a given metric (not all metrics are present in all files)
-            accs = np.array([v.get(task.metric, None) for k, v in raw_results.items() if task.benchmark == k])
-            if accs.size == 0 or any([acc is None for acc in accs]):
-                continue
-
-            mean_acc = np.mean(accs) * 100.0
-            results[task.benchmark] = mean_acc
-
-        return self(
-            eval_name=f"{inference_provider}:{model_id}",
-            model_id=model_id,
-            inference_provider=inference_provider,
-            results=results,
-            date=date,
-        )
-
-    def to_dict(self):
-        """Converts the Eval Result to a dict compatible with our dataframe display"""
-        average = sum([v for v in self.results.values() if v is not None]) / len(Tasks)
-        data_dict = {
-            "eval_name": self.eval_name,  # not a column, just a save name,
-            "model_id": self.model_id,
-            "inference_provider": self.inference_provider,
-            "date": self.date,
-        }
-
-        for task in Tasks:
-            data_dict[task.value.col_name] = self.results[task.value.benchmark]
-
-        return data_dict
-
-
-def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResult]:
-    """From the path of the results folder root, extract all needed info for results"""
-    model_result_filepaths = []
-
-    for root, _, files in os.walk(results_path):
-        # We should only have json files in model results
-        if len(files) == 0 or any([not f.endswith(".json") for f in files]):
-            continue
-
-        # Sort the files by date
-        try:
-            files.sort(key=lambda x: x.removesuffix(".json").removeprefix("results_")[:-7])
-        except dateutil.parser._parser.ParserError:
-            files = [files[-1]]
-
-        for file in files:
-            model_result_filepaths.append(os.path.join(root, file))
-
-    eval_results = {}
-    for model_result_filepath in model_result_filepaths:
-        # Creation of result
-        eval_result = EvalResult.init_from_json_file(model_result_filepath)
-
-        # Store results of same eval together
-        eval_name = eval_result.eval_name
-        if eval_name in eval_results.keys():
-            eval_results[eval_name].results.update({k: v for k, v in eval_result.results.items() if v is not None})
-        else:
-            eval_results[eval_name] = eval_result
-
-    results = []
-    for v in eval_results.values():
-        try:
-            v.to_dict()  # we test if the dict version is complete
-            results.append(v)
-        except KeyError:  # not all eval values present
-            continue
-
-    return results

src/leaderboard/utils.py

@@ -0,0 +1,18 @@
+from dataclasses import dataclass
+
+
+@dataclass
+class ColumnContent:
+    name: str
+    type: str
+    displayed_by_default: bool
+    hidden: bool = False
+    never_hidden: bool = False
+
+
+## Leaderboard columns
+COLUMNS = [
+    ColumnContent("Model", type="str", displayed_by_default=True, never_hidden=True),
+    ColumnContent("Average", type="number", displayed_by_default=True),
+    ColumnContent("RobloxQA", type="number", displayed_by_default=True),
+]

src/populate.py

@@ -1,56 +0,0 @@
-import json
-import os
-
-import pandas as pd
-
-from src.display.formatting import has_no_nan_values, make_clickable_model
-from src.display.utils import AutoEvalColumn, EvalQueueColumn
-from src.leaderboard.read_evals import get_raw_eval_results
-
-
-def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
-    """Creates a dataframe from all the individual experiment results"""
-    raw_data = get_raw_eval_results(results_path, requests_path)
-    all_data_json = [v.to_dict() for v in raw_data]
-
-    df = pd.DataFrame.from_records(all_data_json)
-    # filter out if any of the benchmarks have not been produced
-    df = df[has_no_nan_values(df, benchmark_cols)]
-
-    df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
-    df = df[cols].round(decimals=2)
-
-    return df
-
-
-def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
-    """Creates the different dataframes for the evaluation queues requestes"""
-    entries = [entry for entry in os.listdir(save_path) if not entry.startswith(".")]
-    all_evals = []
-
-    for entry in entries:
-        if ".json" in entry:
-            file_path = os.path.join(save_path, entry)
-            with open(file_path) as fp:
-                data = json.load(fp)
-
-            all_evals.append(data)
-        elif ".md" not in entry:
-            # this is a folder
-            sub_entries = [
-                e for e in os.listdir(f"{save_path}/{entry}") if os.path.isfile(e) and not e.startswith(".")
-            ]
-            for sub_entry in sub_entries:
-                file_path = os.path.join(save_path, entry, sub_entry)
-                with open(file_path) as fp:
-                    data = json.load(fp)
-
-                all_evals.append(data)
-
-    pending_list = [e for e in all_evals if e["status"] in ["PENDING", "RERUN"]]
-    running_list = [e for e in all_evals if e["status"] == "RUNNING"]
-    finished_list = [e for e in all_evals if e["status"].startswith("FINISHED") or e["status"] == "PENDING_NEW_EVAL"]
-    df_pending = pd.DataFrame.from_records(pending_list, columns=cols)
-    df_running = pd.DataFrame.from_records(running_list, columns=cols)
-    df_finished = pd.DataFrame.from_records(finished_list, columns=cols)
-    return df_finished[cols], df_running[cols], df_pending[cols]

src/submission/check_validity.py

@@ -1,36 +0,0 @@
-import json
-import os
-import re
-from collections import defaultdict
-from datetime import datetime, timedelta, timezone
-
-import huggingface_hub
-from huggingface_hub import ModelCard
-from huggingface_hub.hf_api import ModelInfo
-from transformers import AutoConfig
-from transformers.models.auto.tokenization_auto import AutoTokenizer
-
-
-def already_submitted_models(requested_models_dir: str) -> set[str]:
-    """Gather a list of already submitted models to avoid duplicates"""
-    depth = 1
-    file_names = []
-    users_to_submission_dates = defaultdict(list)
-
-    for root, _, files in os.walk(requested_models_dir):
-        current_depth = root.count(os.sep) - requested_models_dir.count(os.sep)
-        if current_depth == depth:
-            for file in files:
-                if not file.endswith(".json"):
-                    continue
-                with open(os.path.join(root, file), "r") as f:
-                    info = json.load(f)
-                    file_names.append(f"{info['inference_provider']}:{info['model_id']}")
-
-                    # Select organisation
-                    if info["model_id"].count("/") == 0 or "submitted_time" not in info:
-                        continue
-                    organisation, _ = info["model_id"].split("/")
-                    users_to_submission_dates[organisation].append(info["submitted_time"])
-
-    return set(file_names), users_to_submission_dates

src/submission/submit.py

@@ -1,70 +0,0 @@
-import json
-import os
-from datetime import datetime, timezone
-
-from src.display.formatting import styled_error, styled_message, styled_warning
-from src.envs import API, EVAL_REQUESTS_PATH, QUEUE_REPO, TOKEN
-from src.submission.check_validity import already_submitted_models
-
-REQUESTED_MODELS = None
-USERS_TO_SUBMISSION_DATES = None
-
-
-def add_new_eval(
-    model_id: str,
-    inference_provider: str,
-):
-    global REQUESTED_MODELS
-    global USERS_TO_SUBMISSION_DATES
-    if not REQUESTED_MODELS:
-        REQUESTED_MODELS, USERS_TO_SUBMISSION_DATES = already_submitted_models(EVAL_REQUESTS_PATH)
-
-    if not inference_provider:
-        return styled_error("Please select an inference provider.")
-
-    user_name = ""
-    model_path = model
-    if "/" in model:
-        user_name = model.split("/")[0]
-        model_path = model.split("/")[1]
-
-    current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
-
-    print("Adding new eval")
-
-    eval_entry = {
-        "submitted_time": current_time,
-        "status": "PENDING",
-        "model_id": model_id,
-        "inference_provider": inference_provider,
-    }
-
-    # Check for duplicate submission
-    if f"{inference_provider}:{model_id}" in REQUESTED_MODELS:
-        return styled_warning("This model has been already submitted.")
-
-    print("Creating eval file")
-    OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
-    os.makedirs(OUT_DIR, exist_ok=True)
-    out_path = f"{OUT_DIR}/{model_path}_eval_request_{inference_provider}.json"
-
-    with open(out_path, "w") as f:
-        f.write(json.dumps(eval_entry))
-
-    print("Uploading eval file")
-    API.upload_file(
-        path_or_fileobj=out_path,
-        path_in_repo=out_path.split("eval-queue/")[1],
-        repo_id=QUEUE_REPO,
-        repo_type="dataset",
-        commit_message=f"Add {model_id} to eval queue",
-    )
-
-    # Remove the local file
-    os.remove(out_path)
-
-    REQUESTED_MODELS.add(f"{inference_provider}:{model_id}")
-
-    return styled_message(
-        "Your request has been submitted to the evaluation queue!\nPlease wait for up to an hour for the model to show in the PENDING list."
-    )