from dataclasses import dataclass, make_dataclass from enum import Enum import pandas as pd from src.about import QATasks, CodeGenTasks def fields(raw_class): return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"] @dataclass class ColumnContent: name: str type: str displayed_by_default: bool hidden: bool = False never_hidden: bool = False # ---- QA Leaderboard columns ---- qa_column_dict = [] qa_column_dict.append(["rank", ColumnContent, ColumnContent("#", "number", True, never_hidden=True)]) qa_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)]) qa_column_dict.append(["size_access", ColumnContent, ColumnContent("Size / Access", "str", True)]) for task in QATasks: qa_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)]) qa_column_dict.append(["delta_overall", ColumnContent, ColumnContent("Delta vs 8B", "number", True)]) QALeaderboardColumn = make_dataclass("QALeaderboardColumn", qa_column_dict, frozen=True) QA_COLS = [c.name for c in fields(QALeaderboardColumn) if not c.hidden] QA_BENCHMARK_COLS = [t.value.col_name for t in QATasks] # ---- Code Generation Leaderboard columns ---- codegen_column_dict = [] codegen_column_dict.append(["rank", ColumnContent, ColumnContent("#", "number", True, never_hidden=True)]) codegen_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)]) codegen_column_dict.append(["method", ColumnContent, ColumnContent("Method", "str", True)]) for task in CodeGenTasks: codegen_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)]) CodeGenLeaderboardColumn = make_dataclass("CodeGenLeaderboardColumn", codegen_column_dict, frozen=True) CODEGEN_COLS = [c.name for c in fields(CodeGenLeaderboardColumn) if not c.hidden] CODEGEN_BENCHMARK_COLS = [t.value.col_name for t in CodeGenTasks] # ---- Model Types (kept for submission compatibility) ---- @dataclass class ModelDetails: name: str display_name: str = "" symbol: str = "" class ModelType(Enum): PT = ModelDetails(name="pretrained", symbol="🟢") FT = ModelDetails(name="fine-tuned", symbol="🔶") IFT = ModelDetails(name="instruction-tuned", symbol="⭕") RL = ModelDetails(name="RL-tuned", symbol="🟦") Unknown = ModelDetails(name="", symbol="?") def to_str(self, separator=" "): return f"{self.value.symbol}{separator}{self.value.name}" @staticmethod def from_str(type): if "fine-tuned" in type or "🔶" in type: return ModelType.FT if "pretrained" in type or "🟢" in type: return ModelType.PT if "RL-tuned" in type or "🟦" in type: return ModelType.RL if "instruction-tuned" in type or "⭕" in type: return ModelType.IFT return ModelType.Unknown class WeightType(Enum): Adapter = ModelDetails("Adapter") Original = ModelDetails("Original") Delta = ModelDetails("Delta") class Precision(Enum): float16 = ModelDetails("float16") bfloat16 = ModelDetails("bfloat16") Unknown = ModelDetails("?") def from_str(precision): if precision in ["torch.float16", "float16"]: return Precision.float16 if precision in ["torch.bfloat16", "bfloat16"]: return Precision.bfloat16 return Precision.Unknown