|
|
""" |
|
|
Display utilities and column definitions for MUSEval Leaderboard |
|
|
""" |
|
|
|
|
|
from dataclasses import dataclass |
|
|
from typing import List, Dict, Any |
|
|
from enum import Enum |
|
|
|
|
|
|
|
|
@dataclass |
|
|
class ModelInfoColumn: |
|
|
name: str |
|
|
type: str = "str" |
|
|
displayed_by_default: bool = True |
|
|
never_hidden: bool = False |
|
|
hidden: bool = False |
|
|
|
|
|
|
|
|
model_info_columns = [ |
|
|
ModelInfoColumn("model", "str", True, True, False), |
|
|
ModelInfoColumn("organization", "str", True, False, False), |
|
|
ModelInfoColumn("submission_date", "str", True, False, False), |
|
|
ModelInfoColumn("task", "str", True, False, False), |
|
|
ModelInfoColumn("dataset_version", "str", True, False, False), |
|
|
ModelInfoColumn("paper_url", "str", False, False, False), |
|
|
ModelInfoColumn("code_url", "str", False, False, False), |
|
|
ModelInfoColumn("domains", "number", True, False, False), |
|
|
ModelInfoColumn("categories", "number", True, False, False), |
|
|
ModelInfoColumn("datasets", "number", True, False, False), |
|
|
] |
|
|
|
|
|
|
|
|
BENCHMARK_COLS = [ |
|
|
"MAE", "Uni-MAE", "RMSE", "MAPE", "R²", "SMAPE", "Uni-Multi" |
|
|
] |
|
|
|
|
|
|
|
|
EVAL_COLS = [ |
|
|
"model", "submitter", "submission_date", "domain", "category", "dataset", |
|
|
"task", "dataset_version", "paper_url", "code_url" |
|
|
] |
|
|
|
|
|
|
|
|
EVAL_TYPES = ["multivariate_forecasting"] |
|
|
|
|
|
|
|
|
class ModelType(Enum): |
|
|
FOUNDATION = "Foundation Model" |
|
|
TRADITIONAL = "Traditional" |
|
|
NEURAL = "Neural Network" |
|
|
TRANSFORMER = "Transformer" |
|
|
|
|
|
|
|
|
class WeightType(Enum): |
|
|
LIGHTWEIGHT = "Lightweight" |
|
|
MEDIUM = "Medium" |
|
|
HEAVY = "Heavy" |
|
|
|
|
|
|
|
|
class Precision(Enum): |
|
|
FLOAT16 = "FP16" |
|
|
FLOAT32 = "FP32" |
|
|
MIXED = "Mixed" |
|
|
|
|
|
|
|
|
def fields(cls): |
|
|
"""Get fields from dataclass""" |
|
|
return cls.__dataclass_fields__.values() if hasattr(cls, '__dataclass_fields__') else [] |
|
|
|