| """All the tasks in the benchmark.""" | |
| from pydantic import BaseModel | |
| class Task(BaseModel): | |
| """Class to hold task information.""" | |
| name: str | |
| metric: str | |
| def __hash__(self): | |
| return hash(self.name) | |
| TEXT_CLASSIFICATION = Task(name="text classification", metric="mcc") | |
| GRAMMAR = Task(name="grammar", metric="mcc") | |
| INFORMATION_EXTRACTION = Task(name="information extraction", metric="micro_f1_no_misc") | |
| READING_COMPREHENSION = Task(name="reading comprehension", metric="em") | |
| SUMMARISATION = Task(name="summarisation", metric="chr_f3pp") | |
| KNOWLEDGE = Task(name="knowledge", metric="mcc") | |
| COMMON_SENSE_REASONING = Task(name="common-sense reasoning", metric="mcc") | |
| ALL_TASKS = [obj for obj in globals().values() if isinstance(obj, Task)] | |