+ {`import braindecode
+from braindecode.models import EEGNetv4
+
+# Load your model
+model = EEGNetv4(
+ n_chans=22,
+ n_outputs=4,
+ n_times=1000,
+)
+# Or from Hugging Face Hub
+# model = braindecode.models.load("your-username/your-model", revision="main")`}
+
+ {citation}
+ {error}
" - - -def styled_warning(warn): - return f"{warn}
" - - -def styled_message(message): - return f"{message}
" - - -def has_no_nan_values(df, columns): - return df[columns].notna().all(axis=1) - - -def has_nan_values(df, columns): - return df[columns].isna().any(axis=1) diff --git a/src/display/utils.py b/src/display/utils.py deleted file mode 100644 index 93df13e57a63ee679f863260185b34a43e4f040d..0000000000000000000000000000000000000000 --- a/src/display/utils.py +++ /dev/null @@ -1,110 +0,0 @@ -from dataclasses import dataclass, make_dataclass -from enum import Enum - -import pandas as pd - -from src.about import Tasks - -def fields(raw_class): - return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"] - - -# These classes are for user facing column names, -# to avoid having to change them all around the code -# when a modif is needed -@dataclass -class ColumnContent: - name: str - type: str - displayed_by_default: bool - hidden: bool = False - never_hidden: bool = False - -## Leaderboard columns -auto_eval_column_dict = [] -# Init -auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)]) -auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)]) -#Scores -auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)]) -for task in Tasks: - auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)]) -# Model information -auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)]) -auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)]) -auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)]) -auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)]) -auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub License", "str", False)]) -auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)]) -auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)]) -auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)]) -auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)]) - -# We use make dataclass to dynamically fill the scores from Tasks -AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True) - -## For the queue columns in the submission tab -@dataclass(frozen=True) -class EvalQueueColumn: # Queue column - model = ColumnContent("model", "markdown", True) - revision = ColumnContent("revision", "str", True) - private = ColumnContent("private", "bool", True) - precision = ColumnContent("precision", "str", True) - weight_type = ColumnContent("weight_type", "str", "Original") - status = ColumnContent("status", "str", True) - -## All the model information that we might need -@dataclass -class ModelDetails: - name: str - display_name: str = "" - symbol: str = "" # emoji - - -class ModelType(Enum): - PT = ModelDetails(name="pretrained", symbol="🟢") - FT = ModelDetails(name="fine-tuned", symbol="🔶") - IFT = ModelDetails(name="instruction-tuned", symbol="⭕") - RL = ModelDetails(name="RL-tuned", symbol="🟦") - Unknown = ModelDetails(name="", symbol="?") - - def to_str(self, separator=" "): - return f"{self.value.symbol}{separator}{self.value.name}" - - @staticmethod - def from_str(type): - if "fine-tuned" in type or "🔶" in type: - return ModelType.FT - if "pretrained" in type or "🟢" in type: - return ModelType.PT - if "RL-tuned" in type or "🟦" in type: - return ModelType.RL - if "instruction-tuned" in type or "⭕" in type: - return ModelType.IFT - return ModelType.Unknown - -class WeightType(Enum): - Adapter = ModelDetails("Adapter") - Original = ModelDetails("Original") - Delta = ModelDetails("Delta") - -class Precision(Enum): - float16 = ModelDetails("float16") - bfloat16 = ModelDetails("bfloat16") - Unknown = ModelDetails("?") - - def from_str(precision): - if precision in ["torch.float16", "float16"]: - return Precision.float16 - if precision in ["torch.bfloat16", "bfloat16"]: - return Precision.bfloat16 - return Precision.Unknown - -# Column selection -COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden] - -EVAL_COLS = [c.name for c in fields(EvalQueueColumn)] -EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)] - -BENCHMARK_COLS = [t.value.col_name for t in Tasks] - diff --git a/src/envs.py b/src/envs.py deleted file mode 100644 index d761858069abf7ff590445e4770c4c3ce08b9222..0000000000000000000000000000000000000000 --- a/src/envs.py +++ /dev/null @@ -1,25 +0,0 @@ -import os - -from huggingface_hub import HfApi - -# Info to change for your repository -# ---------------------------------- -TOKEN = os.environ.get("HF_TOKEN") # A read/write token for your org - -OWNER = "demo-leaderboard-backend" # Change to your org - don't forget to create a results and request dataset, with the correct format! -# ---------------------------------- - -REPO_ID = f"{OWNER}/leaderboard" -QUEUE_REPO = f"{OWNER}/requests" -RESULTS_REPO = f"{OWNER}/results" - -# If you setup a cache later, just change HF_HOME -CACHE_PATH=os.getenv("HF_HOME", ".") - -# Local caches -EVAL_REQUESTS_PATH = os.path.join(CACHE_PATH, "eval-queue") -EVAL_RESULTS_PATH = os.path.join(CACHE_PATH, "eval-results") -EVAL_REQUESTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-queue-bk") -EVAL_RESULTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-results-bk") - -API = HfApi(token=TOKEN) diff --git a/src/leaderboard/read_evals.py b/src/leaderboard/read_evals.py deleted file mode 100644 index f90129f4768433ff52ba083f53cc501128a00430..0000000000000000000000000000000000000000 --- a/src/leaderboard/read_evals.py +++ /dev/null @@ -1,196 +0,0 @@ -import glob -import json -import math -import os -from dataclasses import dataclass - -import dateutil -import numpy as np - -from src.display.formatting import make_clickable_model -from src.display.utils import AutoEvalColumn, ModelType, Tasks, Precision, WeightType -from src.submission.check_validity import is_model_on_hub - - -@dataclass -class EvalResult: - """Represents one full evaluation. Built from a combination of the result and request file for a given run. - """ - eval_name: str # org_model_precision (uid) - full_model: str # org/model (path on hub) - org: str - model: str - revision: str # commit hash, "" if main - results: dict - precision: Precision = Precision.Unknown - model_type: ModelType = ModelType.Unknown # Pretrained, fine tuned, ... - weight_type: WeightType = WeightType.Original # Original or Adapter - architecture: str = "Unknown" - license: str = "?" - likes: int = 0 - num_params: int = 0 - date: str = "" # submission date of request file - still_on_hub: bool = False - - @classmethod - def init_from_json_file(self, json_filepath): - """Inits the result from the specific model result file""" - with open(json_filepath) as fp: - data = json.load(fp) - - config = data.get("config") - - # Precision - precision = Precision.from_str(config.get("model_dtype")) - - # Get model and org - org_and_model = config.get("model_name", config.get("model_args", None)) - org_and_model = org_and_model.split("/", 1) - - if len(org_and_model) == 1: - org = None - model = org_and_model[0] - result_key = f"{model}_{precision.value.name}" - else: - org = org_and_model[0] - model = org_and_model[1] - result_key = f"{org}_{model}_{precision.value.name}" - full_model = "/".join(org_and_model) - - still_on_hub, _, model_config = is_model_on_hub( - full_model, config.get("model_sha", "main"), trust_remote_code=True, test_tokenizer=False - ) - architecture = "?" - if model_config is not None: - architectures = getattr(model_config, "architectures", None) - if architectures: - architecture = ";".join(architectures) - - # Extract results available in this file (some results are split in several files) - results = {} - for task in Tasks: - task = task.value - - # We average all scores of a given metric (not all metrics are present in all files) - accs = np.array([v.get(task.metric, None) for k, v in data["results"].items() if task.benchmark == k]) - if accs.size == 0 or any([acc is None for acc in accs]): - continue - - mean_acc = np.mean(accs) * 100.0 - results[task.benchmark] = mean_acc - - return self( - eval_name=result_key, - full_model=full_model, - org=org, - model=model, - results=results, - precision=precision, - revision= config.get("model_sha", ""), - still_on_hub=still_on_hub, - architecture=architecture - ) - - def update_with_request_file(self, requests_path): - """Finds the relevant request file for the current model and updates info with it""" - request_file = get_request_file_for_model(requests_path, self.full_model, self.precision.value.name) - - try: - with open(request_file, "r") as f: - request = json.load(f) - self.model_type = ModelType.from_str(request.get("model_type", "")) - self.weight_type = WeightType[request.get("weight_type", "Original")] - self.license = request.get("license", "?") - self.likes = request.get("likes", 0) - self.num_params = request.get("params", 0) - self.date = request.get("submitted_time", "") - except Exception: - print(f"Could not find request file for {self.org}/{self.model} with precision {self.precision.value.name}") - - def to_dict(self): - """Converts the Eval Result to a dict compatible with our dataframe display""" - average = sum([v for v in self.results.values() if v is not None]) / len(Tasks) - data_dict = { - "eval_name": self.eval_name, # not a column, just a save name, - AutoEvalColumn.precision.name: self.precision.value.name, - AutoEvalColumn.model_type.name: self.model_type.value.name, - AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol, - AutoEvalColumn.weight_type.name: self.weight_type.value.name, - AutoEvalColumn.architecture.name: self.architecture, - AutoEvalColumn.model.name: make_clickable_model(self.full_model), - AutoEvalColumn.revision.name: self.revision, - AutoEvalColumn.average.name: average, - AutoEvalColumn.license.name: self.license, - AutoEvalColumn.likes.name: self.likes, - AutoEvalColumn.params.name: self.num_params, - AutoEvalColumn.still_on_hub.name: self.still_on_hub, - } - - for task in Tasks: - data_dict[task.value.col_name] = self.results[task.value.benchmark] - - return data_dict - - -def get_request_file_for_model(requests_path, model_name, precision): - """Selects the correct request file for a given model. Only keeps runs tagged as FINISHED""" - request_files = os.path.join( - requests_path, - f"{model_name}_eval_request_*.json", - ) - request_files = glob.glob(request_files) - - # Select correct request file (precision) - request_file = "" - request_files = sorted(request_files, reverse=True) - for tmp_request_file in request_files: - with open(tmp_request_file, "r") as f: - req_content = json.load(f) - if ( - req_content["status"] in ["FINISHED"] - and req_content["precision"] == precision.split(".")[-1] - ): - request_file = tmp_request_file - return request_file - - -def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResult]: - """From the path of the results folder root, extract all needed info for results""" - model_result_filepaths = [] - - for root, _, files in os.walk(results_path): - # We should only have json files in model results - if len(files) == 0 or any([not f.endswith(".json") for f in files]): - continue - - # Sort the files by date - try: - files.sort(key=lambda x: x.removesuffix(".json").removeprefix("results_")[:-7]) - except dateutil.parser._parser.ParserError: - files = [files[-1]] - - for file in files: - model_result_filepaths.append(os.path.join(root, file)) - - eval_results = {} - for model_result_filepath in model_result_filepaths: - # Creation of result - eval_result = EvalResult.init_from_json_file(model_result_filepath) - eval_result.update_with_request_file(requests_path) - - # Store results of same eval together - eval_name = eval_result.eval_name - if eval_name in eval_results.keys(): - eval_results[eval_name].results.update({k: v for k, v in eval_result.results.items() if v is not None}) - else: - eval_results[eval_name] = eval_result - - results = [] - for v in eval_results.values(): - try: - v.to_dict() # we test if the dict version is complete - results.append(v) - except KeyError: # not all eval values present - continue - - return results diff --git a/src/populate.py b/src/populate.py deleted file mode 100644 index 07f0c3af11cdb57f07ddfd29e654d76dd00f5c1b..0000000000000000000000000000000000000000 --- a/src/populate.py +++ /dev/null @@ -1,58 +0,0 @@ -import json -import os - -import pandas as pd - -from src.display.formatting import has_no_nan_values, make_clickable_model -from src.display.utils import AutoEvalColumn, EvalQueueColumn -from src.leaderboard.read_evals import get_raw_eval_results - - -def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame: - """Creates a dataframe from all the individual experiment results""" - raw_data = get_raw_eval_results(results_path, requests_path) - all_data_json = [v.to_dict() for v in raw_data] - - df = pd.DataFrame.from_records(all_data_json) - df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False) - df = df[cols].round(decimals=2) - - # filter out if any of the benchmarks have not been produced - df = df[has_no_nan_values(df, benchmark_cols)] - return df - - -def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]: - """Creates the different dataframes for the evaluation queues requestes""" - entries = [entry for entry in os.listdir(save_path) if not entry.startswith(".")] - all_evals = [] - - for entry in entries: - if ".json" in entry: - file_path = os.path.join(save_path, entry) - with open(file_path) as fp: - data = json.load(fp) - - data[EvalQueueColumn.model.name] = make_clickable_model(data["model"]) - data[EvalQueueColumn.revision.name] = data.get("revision", "main") - - all_evals.append(data) - elif ".md" not in entry: - # this is a folder - sub_entries = [e for e in os.listdir(f"{save_path}/{entry}") if os.path.isfile(e) and not e.startswith(".")] - for sub_entry in sub_entries: - file_path = os.path.join(save_path, entry, sub_entry) - with open(file_path) as fp: - data = json.load(fp) - - data[EvalQueueColumn.model.name] = make_clickable_model(data["model"]) - data[EvalQueueColumn.revision.name] = data.get("revision", "main") - all_evals.append(data) - - pending_list = [e for e in all_evals if e["status"] in ["PENDING", "RERUN"]] - running_list = [e for e in all_evals if e["status"] == "RUNNING"] - finished_list = [e for e in all_evals if e["status"].startswith("FINISHED") or e["status"] == "PENDING_NEW_EVAL"] - df_pending = pd.DataFrame.from_records(pending_list, columns=cols) - df_running = pd.DataFrame.from_records(running_list, columns=cols) - df_finished = pd.DataFrame.from_records(finished_list, columns=cols) - return df_finished[cols], df_running[cols], df_pending[cols] diff --git a/src/submission/check_validity.py b/src/submission/check_validity.py deleted file mode 100644 index d06ee4c444178e369214fbf33d82e81c6f087850..0000000000000000000000000000000000000000 --- a/src/submission/check_validity.py +++ /dev/null @@ -1,99 +0,0 @@ -import json -import os -import re -from collections import defaultdict -from datetime import datetime, timedelta, timezone - -import huggingface_hub -from huggingface_hub import ModelCard -from huggingface_hub.hf_api import ModelInfo -from transformers import AutoConfig -from transformers.models.auto.tokenization_auto import AutoTokenizer - -def check_model_card(repo_id: str) -> tuple[bool, str]: - """Checks if the model card and license exist and have been filled""" - try: - card = ModelCard.load(repo_id) - except huggingface_hub.utils.EntryNotFoundError: - return False, "Please add a model card to your model to explain how you trained/fine-tuned it." - - # Enforce license metadata - if card.data.license is None: - if not ("license_name" in card.data and "license_link" in card.data): - return False, ( - "License not found. Please add a license to your model card using the `license` metadata or a" - " `license_name`/`license_link` pair." - ) - - # Enforce card content - if len(card.text) < 200: - return False, "Please add a description to your model card, it is too short." - - return True, "" - -def is_model_on_hub(model_name: str, revision: str, token: str = None, trust_remote_code=False, test_tokenizer=False) -> tuple[bool, str]: - """Checks if the model model_name is on the hub, and whether it (and its tokenizer) can be loaded with AutoClasses.""" - try: - config = AutoConfig.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token) - if test_tokenizer: - try: - tk = AutoTokenizer.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token) - except ValueError as e: - return ( - False, - f"uses a tokenizer which is not in a transformers release: {e}", - None - ) - except Exception as e: - return (False, "'s tokenizer cannot be loaded. Is your tokenizer class in a stable transformers release, and correctly configured?", None) - return True, None, config - - except ValueError: - return ( - False, - "needs to be launched with `trust_remote_code=True`. For safety reason, we do not allow these models to be automatically submitted to the leaderboard.", - None - ) - - except Exception as e: - return False, "was not found on hub!", None - - -def get_model_size(model_info: ModelInfo, precision: str): - """Gets the model size from the configuration, or the model name if the configuration does not contain the information.""" - try: - model_size = round(model_info.safetensors["total"] / 1e9, 3) - except (AttributeError, TypeError): - return 0 # Unknown model sizes are indicated as 0, see NUMERIC_INTERVALS in app.py - - size_factor = 8 if (precision == "GPTQ" or "gptq" in model_info.modelId.lower()) else 1 - model_size = size_factor * model_size - return model_size - -def get_model_arch(model_info: ModelInfo): - """Gets the model architecture from the configuration""" - return model_info.config.get("architectures", "Unknown") - -def already_submitted_models(requested_models_dir: str) -> set[str]: - """Gather a list of already submitted models to avoid duplicates""" - depth = 1 - file_names = [] - users_to_submission_dates = defaultdict(list) - - for root, _, files in os.walk(requested_models_dir): - current_depth = root.count(os.sep) - requested_models_dir.count(os.sep) - if current_depth == depth: - for file in files: - if not file.endswith(".json"): - continue - with open(os.path.join(root, file), "r") as f: - info = json.load(f) - file_names.append(f"{info['model']}_{info['revision']}_{info['precision']}") - - # Select organisation - if info["model"].count("/") == 0 or "submitted_time" not in info: - continue - organisation, _ = info["model"].split("/") - users_to_submission_dates[organisation].append(info["submitted_time"]) - - return set(file_names), users_to_submission_dates diff --git a/src/submission/submit.py b/src/submission/submit.py deleted file mode 100644 index cac6ea48e803a0af42dabe5226191c769dbec71d..0000000000000000000000000000000000000000 --- a/src/submission/submit.py +++ /dev/null @@ -1,119 +0,0 @@ -import json -import os -from datetime import datetime, timezone - -from src.display.formatting import styled_error, styled_message, styled_warning -from src.envs import API, EVAL_REQUESTS_PATH, TOKEN, QUEUE_REPO -from src.submission.check_validity import ( - already_submitted_models, - check_model_card, - get_model_size, - is_model_on_hub, -) - -REQUESTED_MODELS = None -USERS_TO_SUBMISSION_DATES = None - -def add_new_eval( - model: str, - base_model: str, - revision: str, - precision: str, - weight_type: str, - model_type: str, -): - global REQUESTED_MODELS - global USERS_TO_SUBMISSION_DATES - if not REQUESTED_MODELS: - REQUESTED_MODELS, USERS_TO_SUBMISSION_DATES = already_submitted_models(EVAL_REQUESTS_PATH) - - user_name = "" - model_path = model - if "/" in model: - user_name = model.split("/")[0] - model_path = model.split("/")[1] - - precision = precision.split(" ")[0] - current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") - - if model_type is None or model_type == "": - return styled_error("Please select a model type.") - - # Does the model actually exist? - if revision == "": - revision = "main" - - # Is the model on the hub? - if weight_type in ["Delta", "Adapter"]: - base_model_on_hub, error, _ = is_model_on_hub(model_name=base_model, revision=revision, token=TOKEN, test_tokenizer=True) - if not base_model_on_hub: - return styled_error(f'Base model "{base_model}" {error}') - - if not weight_type == "Adapter": - model_on_hub, error, _ = is_model_on_hub(model_name=model, revision=revision, token=TOKEN, test_tokenizer=True) - if not model_on_hub: - return styled_error(f'Model "{model}" {error}') - - # Is the model info correctly filled? - try: - model_info = API.model_info(repo_id=model, revision=revision) - except Exception: - return styled_error("Could not get your model information. Please fill it up properly.") - - model_size = get_model_size(model_info=model_info, precision=precision) - - # Were the model card and license filled? - try: - license = model_info.cardData["license"] - except Exception: - return styled_error("Please select a license for your model") - - modelcard_OK, error_msg = check_model_card(model) - if not modelcard_OK: - return styled_error(error_msg) - - # Seems good, creating the eval - print("Adding new eval") - - eval_entry = { - "model": model, - "base_model": base_model, - "revision": revision, - "precision": precision, - "weight_type": weight_type, - "status": "PENDING", - "submitted_time": current_time, - "model_type": model_type, - "likes": model_info.likes, - "params": model_size, - "license": license, - "private": False, - } - - # Check for duplicate submission - if f"{model}_{revision}_{precision}" in REQUESTED_MODELS: - return styled_warning("This model has been already submitted.") - - print("Creating eval file") - OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}" - os.makedirs(OUT_DIR, exist_ok=True) - out_path = f"{OUT_DIR}/{model_path}_eval_request_False_{precision}_{weight_type}.json" - - with open(out_path, "w") as f: - f.write(json.dumps(eval_entry)) - - print("Uploading eval file") - API.upload_file( - path_or_fileobj=out_path, - path_in_repo=out_path.split("eval-queue/")[1], - repo_id=QUEUE_REPO, - repo_type="dataset", - commit_message=f"Add {model} to eval queue", - ) - - # Remove the local file - os.remove(out_path) - - return styled_message( - "Your request has been submitted to the evaluation queue!\nPlease wait for up to an hour for the model to show in the PENDING list." - )