VibeHash's picture
Release laya-ledgar, run ce-seed0
779ff0b verified
Raw History Blame Contribute Delete
3.51 kB
"""What makes one task different from another: its source, its one question, and its training settings.
Every step (data, fit, train, evaluate, package) reads a `Task` and nothing else that is task-specific, so a new
task is one module under `layatasks/tasks/` and the steps stay the same. A task module defines `TASK`, and may define
`extra_scores` for numbers its benchmark reports beyond accuracy and calibration.
"""
from __future__ import annotations
import importlib
import pkgutil
from collections import Counter
from dataclasses import dataclass
from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple
from layatasks import tasks as _task_modules
# Every task module present. A release ships only its own, so this is read from the folder, not written down.
TASKS = tuple(sorted(m.name for m in pkgutil.iter_modules(_task_modules.__path__)))
@dataclass(frozen=True)
class Task:
name: str
title: str
# The source, pinned by commit and by the SHA-256 of each file read from it.
repo: str
revision: str
license: str
attribution: str
files: Dict[str, Tuple[str, str]]
text_column: str
label_column: str
# The one question every row gets.
question_id: str
instructions: str
option_text: Callable[[str], str]
# SHA-256 of test.jsonl as `layatasks.data` writes it. Set once, on the first build, before any model is scored.
test_sha256: Optional[str]
# Training settings, fixed before any fine-tuned model of this task is scored.
head_max_len: int = 512
max_len: int = 512
target_error: float = 0.01
epochs: int = 4
# Laya's loss: "rlcd", its default, or "soft-ce", plain cross-entropy over the options.
loss: str = "rlcd"
# Training rows kept out of training, to judge a run before the test is read. 0 keeps none out.
dev_rows: int = 0
# The model id the release is uploaded as; `OWNER` is filled in by the package step.
release_name: str = ""
def options(self, names: Sequence[str]) -> List[str]:
return [self.option_text(n) for n in names]
def question(self, names: Sequence[str]) -> Dict[str, Any]:
"""The question every row asks: pick one option, in the source's label order."""
options = self.options(names)
if len(set(options)) != len(options):
dupes = sorted(o for o, c in Counter(options).items() if c > 1)
raise ValueError("two labels read the same once written as text: %r" % (dupes,))
return {"type": "choice", "instructions": self.instructions, "criteria": options}
def rows(self, records: Sequence[Dict[str, Any]], names: Sequence[str]) -> List[Dict[str, Any]]:
"""Laya's `{state, questions, expected}` rows, one a record, in the source's order."""
q = self.question(names)
options = q["criteria"]
return [{"state": r[self.text_column], "questions": {self.question_id: q},
"expected": {self.question_id: options[r[self.label_column]]}} for r in records]
def load(name: str):
"""The task module for `name`. Imported on demand, so a release can ship with only its own task."""
if name not in TASKS:
raise ValueError("unknown task %r; known: %s" % (name, ", ".join(TASKS)))
return importlib.import_module("layatasks.tasks." + name)
def extra_scores(module, preds: List[Dict[str, Any]], gate: Optional[float]) -> Dict[str, Any]:
fn = getattr(module, "extra_scores", None)
return fn(preds, gate) if fn else {}