src/display/utils.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# flake8: noqa E501

from dataclasses import dataclass, field, make_dataclass
from enum import Enum
from functools import partial

import pandas as pd

from src.about import Tasks


def fields(raw_class):
    return [v for k, v in raw_class.__dict__.items() if not k.startswith("__") and not k.endswith("__")]

# These classes are for user facing column names,
# to avoid having to change them all around the code
# when a modif is needed


@dataclass
class ColumnContent:
    name: str
    type: str = field(default='str')
    displayed_by_default: bool = field(default=False)
    hidden: bool = field(default=False)
    never_hidden: bool = field(default=False)


# Helper function to create a ColumnContent with default_factory
def create_column_content(name, type='str', displayed_by_default=False, hidden=False, never_hidden=False):
    return field(default_factory=lambda: ColumnContent(name, type, displayed_by_default, hidden, never_hidden))


auto_eval_column_dict = [
    ("model_type_symbol", ColumnContent, create_column_content(
        "", "str", True, never_hidden=True)),
    ("model", ColumnContent, create_column_content(
        "Model", "markdown", True, never_hidden=True)),
    ("soliditybench", ColumnContent, create_column_content("Score", "number", True)),
    # ("average", ColumnContent, create_column_content("Average", "number", True)),
]

# Add task-specific columns
for task in Tasks:
    displayed_by_default = False
    # Only show NaïveJudge and pass@3 by default
    if task.name in ['task00', 'task02']:  # NaïveJudge and pass@3
        displayed_by_default = True
    auto_eval_column_dict.append((
        task.name,
        ColumnContent,
        create_column_content(
            task.value.col_name,
            'number',
            displayed_by_default,
        ),
    ))

# Add model information columns
hide = True
display = True
model_info_columns = [
    ("model_type", "Type", "str", not display, not hide),
    ("architecture", "Architecture", "str", not display, not hide),
    ("weight_type", "Weight type", "str", not display, hide),
    ("precision", "Precision", "str", not display, not hide),
    ("license", "License", "str", not display, not hide),
    ("params", "Parameters (billions)", "number", not display, not hide),
    ("likes", "Likes", "number", not display, hide),
    ("still_on_hub", "HuggingFace Hub", "bool", not display, hide),
    ("revision", "Revision", "str", not display, not hide),
]

for col_name, display_name, col_type, displayed_by_default, *args in model_info_columns:
    hidden = args[0] if args else False
    auto_eval_column_dict.append((col_name, ColumnContent, create_column_content(
        display_name, col_type, displayed_by_default, hidden)))

# Create the AutoEvalColumn dataclass
AutoEvalColumn = make_dataclass(
    "AutoEvalColumn", auto_eval_column_dict, frozen=True)()

# For the requests columns in the submission tab


@dataclass(frozen=True)
class EvalQueueColumn:  # Queue column
    model = ColumnContent("model", "markdown", True)
    revision = ColumnContent("revision", "str", True)
    # private = ColumnContent("private", "bool", True)
    # precision = ColumnContent("precision", "str", True)
    # weight_type = ColumnContent("weight_type", "str", "Original")
    # status = ColumnContent("status", "str", True)


# All the model information that we might need
@dataclass
class ModelDetails:
    name: str
    display_name: str = ""
    symbol: str = ""  # emoji


class ModelType(Enum):
    PT = ModelDetails(name="pretrained", symbol="💎")
    FT = ModelDetails(name="finetuned", symbol="💍")
    BrainDAO = ModelDetails(name="braindao", symbol="🧠")
    Unknown = ModelDetails(name="", symbol="❓")

    def to_str(self, separator="  "):
        return f"{self.value.symbol}{separator}{self.value.name}"

    @staticmethod
    def from_str(type):
        if "finetuned" in type or "💍" in type:
            return ModelType.FT
        if "pretrained" in type or "💎" in type:
            return ModelType.PT
        if "braindao" in type or "🧠" in type:
            return ModelType.BrainDAO
        return ModelType.Unknown


class WeightType(Enum):
    Adapter = ModelDetails("Adapter")
    Original = ModelDetails("Original")
    Delta = ModelDetails("Delta")


class Precision(Enum):
    float16 = ModelDetails("float16")
    bfloat16 = ModelDetails("bfloat16")
    qt_8bit = ModelDetails("8bit")
    qt_4bit = ModelDetails("4bit")
    qt_GPTQ = ModelDetails("GPTQ")
    Unknown = ModelDetails("Unknown")

    @staticmethod
    def from_str(precision):
        if precision in ["torch.float16", "float16"]:
            return Precision.float16
        if precision in ["torch.bfloat16", "bfloat16"]:
            return Precision.bfloat16
        if precision in ["8bit"]:
            return Precision.qt_8bit
        if precision in ["4bit"]:
            return Precision.qt_4bit
        if precision in ["GPTQ", "None"]:
            return Precision.qt_GPTQ
        return Precision.Unknown


# Column selection
COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]

EVAL_COLS = [c.name for c in fields(EvalQueueColumn)]
EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)]

BENCHMARK_COLS = [t.value.col_name for t in Tasks]