VibeHash's picture
Release laya-ledgar, run ce-seed0
779ff0b verified
Raw History Blame Contribute Delete
7.14 kB
"""Check how a task's rows fit Laya's token budget, and estimate how long its fine-tune takes. CPU only.
Laya puts the question first and the text after it, in one sequence of at most `max_len` tokens. Whatever the
question's options leave over is the room for the text, and a longer text loses its end silently. This step counts
how many training and test rows lose words that way, so the card can say so, and records which test rows they are,
so the scorer can report accuracy on cut and uncut rows apart.
The time estimate replays Laya's own batching (the same calibration hold-out, the same shuffle per epoch, micro-
batches of 8 padded to their longest row) and scales the padded tokens against one measured run: Banking77 on an
RTX 4070 SUPER. It treats time as proportional to padded tokens, which is roughly right up to 512 tokens and
optimistic beyond. It is an estimate, not a measurement.
Run it with `python -m layatasks.fit <task>`.
"""
from __future__ import annotations
import argparse
import random
import statistics
from typing import Any, Dict, List, Sequence
from layatasks import task as tasks
from layatasks.common import data_dir, read_json, read_jsonl, write_json
# Laya's defaults (laya.train.TrainConfig) that decide which rows train and how they are batched.
MICRO_BATCH = 8
CALIB_MAX = 400
CALIB_FRAC = 0.1
CALIB_SEED = 20260922
# The one measured run every estimate is scaled from: laya-banking77 seed 0, 4 epochs, head_max_len 512,
# max_len 512, on an RTX 4070 SUPER, 2026-11-13. Its padded tokens come from `padded_tokens` on its train.jsonl.
REFERENCE = {"name": "laya-banking77 seed 0", "gpu": "RTX 4070 SUPER", "seconds": 1584.7,
"padded_tokens": 14818686}
def train_rows(n: int) -> List[int]:
"""Indices of the rows Laya trains on: all but its calibration hold-out, the same split `laya.train` makes."""
n_calib = min(CALIB_MAX, int(n * CALIB_FRAC))
order = list(range(n))
random.Random(CALIB_SEED).shuffle(order)
held = set(order[:n_calib])
return [i for i in range(n) if i not in held]
def padded_tokens(lengths: Sequence[int], epochs: int, seed: int = 0, micro_batch: int = MICRO_BATCH) -> int:
"""Tokens the GPU works through in a fine-tune, padding included: Laya pads each micro-batch to its longest row."""
keep = [lengths[i] for i in train_rows(len(lengths))]
total = 0
for epoch in range(epochs):
order = list(keep)
random.Random(seed + epoch).shuffle(order)
for start in range(0, len(order), micro_batch):
chunk = order[start:start + micro_batch]
total += max(chunk) * len(chunk)
return total
def head_of(task, names: Sequence[str], tok):
"""`(head token ids, option stats)` for the task's question, built the way inference builds it."""
from laya.common import build_head
from laya.train import to_internal
q = to_internal(task.question_id, task.question(names))
ids, _markers, stats = build_head(tok, q, task.head_max_len)
return q, ids, stats
def state_tokens(tok, text: str) -> int:
from laya.common import encode_text, serialize_state
return len(encode_text(tok, serialize_state(text).replace(tok.mask_token, " "),
add_special_tokens=False)["input_ids"])
def describe(lengths: Sequence[int], room: int) -> Dict[str, Any]:
s = sorted(lengths)
cut = sum(1 for n in s if n > room)
return {"rows": len(s), "state_tokens": {"median": statistics.median(s), "p90": s[int(0.9 * (len(s) - 1))],
"p99": s[int(0.99 * (len(s) - 1))], "max": s[-1]},
"room": room, "truncated": cut, "truncated_share": cut / len(s)}
def check(task, tok, rows: List[Dict[str, Any]], names: Sequence[str], spot: int = 3) -> Dict[str, Any]:
"""Lengths of every row under the task's budgets. The first `spot` rows are rebuilt in full by Laya's
`build_sequence` and must agree with the shortcut used for the rest."""
from laya.common import build_sequence
q, head, stats = head_of(task, names, tok)
room = max(0, task.max_len - len(head) - 1)
states = [state_tokens(tok, r["state"]) for r in rows]
seq = [len(head) + min(n, room) + 1 for n in states]
for r, n in list(zip(rows, seq))[:spot]:
ids, _ = build_sequence(tok, r["state"], q, task.max_len, task.head_max_len)
if len(ids) != n:
raise RuntimeError("length shortcut disagrees with build_sequence: %d vs %d" % (n, len(ids)))
return {"head_tokens": len(head), "options": stats.get("options"), "options_distinct": stats.get("options_distinct"),
"tokens_per_option": stats.get("tokens_per_option"), "states": states, "seq": seq, "room": room}
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
parser.add_argument("task", choices=tasks.TASKS)
parser.add_argument("--base", default=None, help="a local copy of the base checkpoint (default: download it)")
args = parser.parse_args()
from transformers import AutoTokenizer
from layatasks.train import base_dir
task = tasks.load(args.task).TASK
folder = data_dir(task.name)
names = read_json(folder / "labels.json")["names"]
tok = AutoTokenizer.from_pretrained(str(base_dir(args.base)) + "/tokenizer")
train = check(task, tok, read_jsonl(folder / "train.jsonl"), names)
test = check(task, tok, read_jsonl(folder / "test.jsonl"), names)
padded = padded_tokens(train["seq"], task.epochs)
result = {
"task": task.name, "max_len": task.max_len, "head_max_len": task.head_max_len,
"head_tokens": train["head_tokens"], "options": train["options"],
"options_distinct": train["options_distinct"], "tokens_per_option": train["tokens_per_option"],
"train": describe(train["states"], train["room"]),
"test": dict(describe(test["states"], test["room"]),
truncated_rows=[i for i, n in enumerate(test["states"]) if n > test["room"]]),
"estimate": {"epochs": task.epochs, "padded_tokens": padded, "reference": REFERENCE,
"minutes": (REFERENCE["seconds"] * padded / REFERENCE["padded_tokens"] / 60
if REFERENCE["padded_tokens"] else None)},
}
write_json(folder / "fit.json", result)
print("%s: head %d tokens for %d options (%s distinct); room for the text %d tokens"
% (task.name, result["head_tokens"], result["options"], result["options_distinct"], train["room"]))
for split in ("train", "test"):
d = result[split]
print(" %s: %d rows, text tokens median %s, p99 %d, max %d; %d cut short (%.1f%%)"
% (split, d["rows"], d["state_tokens"]["median"], d["state_tokens"]["p99"], d["state_tokens"]["max"],
d["truncated"], 100 * d["truncated_share"]))
if result["estimate"]["minutes"] is not None:
print(" estimated fine-tune: %.0f minutes on the %s (scaled from %s, not measured)"
% (result["estimate"]["minutes"], REFERENCE["gpu"], REFERENCE["name"]))
if __name__ == "__main__":
main()