"""What makes one task different from another: its source, its one question, and its training settings. Every step (data, fit, train, evaluate, package) reads a `Task` and nothing else that is task-specific, so a new task is one module under `layatasks/tasks/` and the steps stay the same. A task module defines `TASK`, and may define `extra_scores` for numbers its benchmark reports beyond accuracy and calibration. """ from __future__ import annotations import importlib import pkgutil from collections import Counter from dataclasses import dataclass from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple from layatasks import tasks as _task_modules # Every task module present. A release ships only its own, so this is read from the folder, not written down. TASKS = tuple(sorted(m.name for m in pkgutil.iter_modules(_task_modules.__path__))) @dataclass(frozen=True) class Task: name: str title: str # The source, pinned by commit and by the SHA-256 of each file read from it. repo: str revision: str license: str attribution: str files: Dict[str, Tuple[str, str]] text_column: str label_column: str # The one question every row gets. question_id: str instructions: str option_text: Callable[[str], str] # SHA-256 of test.jsonl as `layatasks.data` writes it. Set once, on the first build, before any model is scored. test_sha256: Optional[str] # Training settings, fixed before any fine-tuned model of this task is scored. head_max_len: int = 512 max_len: int = 512 target_error: float = 0.01 epochs: int = 4 # Laya's loss: "rlcd", its default, or "soft-ce", plain cross-entropy over the options. loss: str = "rlcd" # Training rows kept out of training, to judge a run before the test is read. 0 keeps none out. dev_rows: int = 0 # The model id the release is uploaded as; `OWNER` is filled in by the package step. release_name: str = "" def options(self, names: Sequence[str]) -> List[str]: return [self.option_text(n) for n in names] def question(self, names: Sequence[str]) -> Dict[str, Any]: """The question every row asks: pick one option, in the source's label order.""" options = self.options(names) if len(set(options)) != len(options): dupes = sorted(o for o, c in Counter(options).items() if c > 1) raise ValueError("two labels read the same once written as text: %r" % (dupes,)) return {"type": "choice", "instructions": self.instructions, "criteria": options} def rows(self, records: Sequence[Dict[str, Any]], names: Sequence[str]) -> List[Dict[str, Any]]: """Laya's `{state, questions, expected}` rows, one a record, in the source's order.""" q = self.question(names) options = q["criteria"] return [{"state": r[self.text_column], "questions": {self.question_id: q}, "expected": {self.question_id: options[r[self.label_column]]}} for r in records] def load(name: str): """The task module for `name`. Imported on demand, so a release can ship with only its own task.""" if name not in TASKS: raise ValueError("unknown task %r; known: %s" % (name, ", ".join(TASKS))) return importlib.import_module("layatasks.tasks." + name) def extra_scores(module, preds: List[Dict[str, Any]], gate: Optional[float]) -> Dict[str, Any]: fn = getattr(module, "extra_scores", None) return fn(preds, gate) if fn else {}