update utils
Browse files- src/display/utils.py +11 -90
src/display/utils.py
CHANGED
|
@@ -13,7 +13,7 @@ class ColumnContent:
|
|
| 13 |
label: str
|
| 14 |
description: str
|
| 15 |
hidden: bool = False
|
| 16 |
-
displayed_by_default: bool = True
|
| 17 |
never_hidden: bool = False
|
| 18 |
|
| 19 |
# Initialize the list of columns for the leaderboard
|
|
@@ -46,7 +46,7 @@ for task in Tasks:
|
|
| 46 |
type=float,
|
| 47 |
label=f"{task.value.col_name} (%)",
|
| 48 |
description=f"Accuracy on {task.value.col_name}",
|
| 49 |
-
displayed_by_default=
|
| 50 |
)
|
| 51 |
)
|
| 52 |
|
|
@@ -57,126 +57,47 @@ COLUMNS.extend([
|
|
| 57 |
type=str,
|
| 58 |
label="Model Type",
|
| 59 |
description="Type of the model (e.g., Transformer, RNN, etc.)",
|
| 60 |
-
displayed_by_default=
|
| 61 |
-
),
|
| 62 |
-
ColumnContent(
|
| 63 |
-
name="architecture",
|
| 64 |
-
type=str,
|
| 65 |
-
label="Architecture",
|
| 66 |
-
description="Model architecture",
|
| 67 |
-
displayed_by_default=False,
|
| 68 |
),
|
| 69 |
ColumnContent(
|
| 70 |
name="weight_type",
|
| 71 |
type=str,
|
| 72 |
label="Weight Type",
|
| 73 |
description="Type of model weights (e.g., Original, Delta, Adapter)",
|
| 74 |
-
displayed_by_default=
|
| 75 |
),
|
| 76 |
ColumnContent(
|
| 77 |
name="precision",
|
| 78 |
type=str,
|
| 79 |
label="Precision",
|
| 80 |
description="Precision of the model weights (e.g., float16)",
|
| 81 |
-
displayed_by_default=
|
| 82 |
),
|
| 83 |
ColumnContent(
|
| 84 |
name="license",
|
| 85 |
type=str,
|
| 86 |
label="License",
|
| 87 |
description="License of the model",
|
| 88 |
-
displayed_by_default=
|
| 89 |
-
),
|
| 90 |
-
ColumnContent(
|
| 91 |
-
name="params",
|
| 92 |
-
type=float,
|
| 93 |
-
label="Parameters (B)",
|
| 94 |
-
description="Number of model parameters in billions",
|
| 95 |
-
displayed_by_default=False,
|
| 96 |
),
|
| 97 |
ColumnContent(
|
| 98 |
name="likes",
|
| 99 |
type=int,
|
| 100 |
label="Likes",
|
| 101 |
description="Number of likes on the Hugging Face Hub",
|
| 102 |
-
displayed_by_default=
|
| 103 |
),
|
| 104 |
ColumnContent(
|
| 105 |
name="still_on_hub",
|
| 106 |
type=bool,
|
| 107 |
label="Available on the Hub",
|
| 108 |
description="Whether the model is still available on the Hugging Face Hub",
|
| 109 |
-
displayed_by_default=
|
| 110 |
-
),
|
| 111 |
-
ColumnContent(
|
| 112 |
-
name="revision",
|
| 113 |
-
type=str,
|
| 114 |
-
label="Model Revision",
|
| 115 |
-
description="Model revision or commit hash",
|
| 116 |
-
displayed_by_default=False,
|
| 117 |
),
|
| 118 |
])
|
| 119 |
|
| 120 |
# Now we can create lists of column names for use in the application
|
| 121 |
COLS = [col.name for col in COLUMNS]
|
| 122 |
-
BENCHMARK_COLS = [col.name for col in COLUMNS if col.name not in [
|
| 123 |
-
|
| 124 |
-
|
| 125 |
-
@dataclass(frozen=True)
|
| 126 |
-
class EvalQueueColumn:
|
| 127 |
-
model: str
|
| 128 |
-
revision: str
|
| 129 |
-
private: bool
|
| 130 |
-
precision: str
|
| 131 |
-
weight_type: str
|
| 132 |
-
status: str
|
| 133 |
-
|
| 134 |
-
EVAL_COLS = ["model", "revision", "private", "precision", "weight_type", "status"]
|
| 135 |
-
EVAL_TYPES = [str, str, bool, str, str, str]
|
| 136 |
-
|
| 137 |
-
## All the model information that we might need
|
| 138 |
-
@dataclass
|
| 139 |
-
class ModelDetails:
|
| 140 |
-
name: str
|
| 141 |
-
display_name: str = ""
|
| 142 |
-
symbol: str = "" # emoji
|
| 143 |
-
|
| 144 |
-
class ModelType(Enum):
|
| 145 |
-
PT = ModelDetails(name="pretrained", symbol="🟢")
|
| 146 |
-
FT = ModelDetails(name="fine-tuned", symbol="🔶")
|
| 147 |
-
IFT = ModelDetails(name="instruction-tuned", symbol="⭕")
|
| 148 |
-
RL = ModelDetails(name="RL-tuned", symbol="🟦")
|
| 149 |
-
Unknown = ModelDetails(name="", symbol="?")
|
| 150 |
-
|
| 151 |
-
def to_str(self, separator=" "):
|
| 152 |
-
return f"{self.value.symbol}{separator}{self.value.name}"
|
| 153 |
-
|
| 154 |
-
@staticmethod
|
| 155 |
-
def from_str(type_str):
|
| 156 |
-
if "fine-tuned" in type_str or "🔶" in type_str:
|
| 157 |
-
return ModelType.FT
|
| 158 |
-
if "pretrained" in type_str or "🟢" in type_str:
|
| 159 |
-
return ModelType.PT
|
| 160 |
-
if "RL-tuned" in type_str or "🟦" in type_str:
|
| 161 |
-
return ModelType.RL
|
| 162 |
-
if "instruction-tuned" in type_str or "⭕" in type_str:
|
| 163 |
-
return ModelType.IFT
|
| 164 |
-
return ModelType.Unknown
|
| 165 |
-
|
| 166 |
-
class WeightType(Enum):
|
| 167 |
-
Adapter = "Adapter"
|
| 168 |
-
Original = "Original"
|
| 169 |
-
Delta = "Delta"
|
| 170 |
-
|
| 171 |
-
class Precision(Enum):
|
| 172 |
-
float16 = "float16"
|
| 173 |
-
bfloat16 = "bfloat16"
|
| 174 |
-
Unknown = "Unknown"
|
| 175 |
-
|
| 176 |
-
@staticmethod
|
| 177 |
-
def from_str(precision_str):
|
| 178 |
-
if precision_str in ["torch.float16", "float16"]:
|
| 179 |
-
return Precision.float16
|
| 180 |
-
if precision_str in ["torch.bfloat16", "bfloat16"]:
|
| 181 |
-
return Precision.bfloat16
|
| 182 |
-
return Precision.Unknown
|
|
|
|
| 13 |
label: str
|
| 14 |
description: str
|
| 15 |
hidden: bool = False
|
| 16 |
+
displayed_by_default: bool = True # All columns displayed by default
|
| 17 |
never_hidden: bool = False
|
| 18 |
|
| 19 |
# Initialize the list of columns for the leaderboard
|
|
|
|
| 46 |
type=float,
|
| 47 |
label=f"{task.value.col_name} (%)",
|
| 48 |
description=f"Accuracy on {task.value.col_name}",
|
| 49 |
+
displayed_by_default=True,
|
| 50 |
)
|
| 51 |
)
|
| 52 |
|
|
|
|
| 57 |
type=str,
|
| 58 |
label="Model Type",
|
| 59 |
description="Type of the model (e.g., Transformer, RNN, etc.)",
|
| 60 |
+
displayed_by_default=True,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 61 |
),
|
| 62 |
ColumnContent(
|
| 63 |
name="weight_type",
|
| 64 |
type=str,
|
| 65 |
label="Weight Type",
|
| 66 |
description="Type of model weights (e.g., Original, Delta, Adapter)",
|
| 67 |
+
displayed_by_default=True,
|
| 68 |
),
|
| 69 |
ColumnContent(
|
| 70 |
name="precision",
|
| 71 |
type=str,
|
| 72 |
label="Precision",
|
| 73 |
description="Precision of the model weights (e.g., float16)",
|
| 74 |
+
displayed_by_default=True,
|
| 75 |
),
|
| 76 |
ColumnContent(
|
| 77 |
name="license",
|
| 78 |
type=str,
|
| 79 |
label="License",
|
| 80 |
description="License of the model",
|
| 81 |
+
displayed_by_default=True,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 82 |
),
|
| 83 |
ColumnContent(
|
| 84 |
name="likes",
|
| 85 |
type=int,
|
| 86 |
label="Likes",
|
| 87 |
description="Number of likes on the Hugging Face Hub",
|
| 88 |
+
displayed_by_default=True,
|
| 89 |
),
|
| 90 |
ColumnContent(
|
| 91 |
name="still_on_hub",
|
| 92 |
type=bool,
|
| 93 |
label="Available on the Hub",
|
| 94 |
description="Whether the model is still available on the Hugging Face Hub",
|
| 95 |
+
displayed_by_default=True,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 96 |
),
|
| 97 |
])
|
| 98 |
|
| 99 |
# Now we can create lists of column names for use in the application
|
| 100 |
COLS = [col.name for col in COLUMNS]
|
| 101 |
+
BENCHMARK_COLS = [col.name for col in COLUMNS if col.name not in [
|
| 102 |
+
"model", "average", "model_type", "weight_type", "precision", "license", "likes", "still_on_hub"
|
| 103 |
+
]]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|