File size: 760 Bytes
d288ea3 317dd0a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 | """All the tasks in the benchmark."""
from pydantic import BaseModel
class Task(BaseModel):
"""Class to hold task information."""
name: str
metric: str
def __hash__(self):
return hash(self.name)
TEXT_CLASSIFICATION = Task(name="text classification", metric="mcc")
GRAMMAR = Task(name="grammar", metric="mcc")
INFORMATION_EXTRACTION = Task(name="information extraction", metric="micro_f1_no_misc")
READING_COMPREHENSION = Task(name="reading comprehension", metric="em")
SUMMARISATION = Task(name="summarisation", metric="chr_f3pp")
KNOWLEDGE = Task(name="knowledge", metric="mcc")
COMMON_SENSE_REASONING = Task(name="common-sense reasoning", metric="mcc")
ALL_TASKS = [obj for obj in globals().values() if isinstance(obj, Task)]
|