File size: 7,143 Bytes
779ff0b
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
"""Check how a task's rows fit Laya's token budget, and estimate how long its fine-tune takes. CPU only.

Laya puts the question first and the text after it, in one sequence of at most `max_len` tokens. Whatever the
question's options leave over is the room for the text, and a longer text loses its end silently. This step counts
how many training and test rows lose words that way, so the card can say so, and records which test rows they are,
so the scorer can report accuracy on cut and uncut rows apart.

The time estimate replays Laya's own batching (the same calibration hold-out, the same shuffle per epoch, micro-
batches of 8 padded to their longest row) and scales the padded tokens against one measured run: Banking77 on an
RTX 4070 SUPER. It treats time as proportional to padded tokens, which is roughly right up to 512 tokens and
optimistic beyond. It is an estimate, not a measurement.

Run it with `python -m layatasks.fit <task>`.
"""

from __future__ import annotations

import argparse
import random
import statistics
from typing import Any, Dict, List, Sequence

from layatasks import task as tasks
from layatasks.common import data_dir, read_json, read_jsonl, write_json

# Laya's defaults (laya.train.TrainConfig) that decide which rows train and how they are batched.
MICRO_BATCH = 8
CALIB_MAX = 400
CALIB_FRAC = 0.1
CALIB_SEED = 20260922

# The one measured run every estimate is scaled from: laya-banking77 seed 0, 4 epochs, head_max_len 512,
# max_len 512, on an RTX 4070 SUPER, 2026-11-13. Its padded tokens come from `padded_tokens` on its train.jsonl.
REFERENCE = {"name": "laya-banking77 seed 0", "gpu": "RTX 4070 SUPER", "seconds": 1584.7,
             "padded_tokens": 14818686}


def train_rows(n: int) -> List[int]:
    """Indices of the rows Laya trains on: all but its calibration hold-out, the same split `laya.train` makes."""
    n_calib = min(CALIB_MAX, int(n * CALIB_FRAC))
    order = list(range(n))
    random.Random(CALIB_SEED).shuffle(order)
    held = set(order[:n_calib])
    return [i for i in range(n) if i not in held]


def padded_tokens(lengths: Sequence[int], epochs: int, seed: int = 0, micro_batch: int = MICRO_BATCH) -> int:
    """Tokens the GPU works through in a fine-tune, padding included: Laya pads each micro-batch to its longest row."""
    keep = [lengths[i] for i in train_rows(len(lengths))]
    total = 0
    for epoch in range(epochs):
        order = list(keep)
        random.Random(seed + epoch).shuffle(order)
        for start in range(0, len(order), micro_batch):
            chunk = order[start:start + micro_batch]
            total += max(chunk) * len(chunk)
    return total


def head_of(task, names: Sequence[str], tok):
    """`(head token ids, option stats)` for the task's question, built the way inference builds it."""
    from laya.common import build_head
    from laya.train import to_internal

    q = to_internal(task.question_id, task.question(names))
    ids, _markers, stats = build_head(tok, q, task.head_max_len)
    return q, ids, stats


def state_tokens(tok, text: str) -> int:
    from laya.common import encode_text, serialize_state

    return len(encode_text(tok, serialize_state(text).replace(tok.mask_token, " "),
                           add_special_tokens=False)["input_ids"])


def describe(lengths: Sequence[int], room: int) -> Dict[str, Any]:
    s = sorted(lengths)
    cut = sum(1 for n in s if n > room)
    return {"rows": len(s), "state_tokens": {"median": statistics.median(s), "p90": s[int(0.9 * (len(s) - 1))],
                                             "p99": s[int(0.99 * (len(s) - 1))], "max": s[-1]},
            "room": room, "truncated": cut, "truncated_share": cut / len(s)}


def check(task, tok, rows: List[Dict[str, Any]], names: Sequence[str], spot: int = 3) -> Dict[str, Any]:
    """Lengths of every row under the task's budgets. The first `spot` rows are rebuilt in full by Laya's
    `build_sequence` and must agree with the shortcut used for the rest."""
    from laya.common import build_sequence

    q, head, stats = head_of(task, names, tok)
    room = max(0, task.max_len - len(head) - 1)
    states = [state_tokens(tok, r["state"]) for r in rows]
    seq = [len(head) + min(n, room) + 1 for n in states]
    for r, n in list(zip(rows, seq))[:spot]:
        ids, _ = build_sequence(tok, r["state"], q, task.max_len, task.head_max_len)
        if len(ids) != n:
            raise RuntimeError("length shortcut disagrees with build_sequence: %d vs %d" % (n, len(ids)))
    return {"head_tokens": len(head), "options": stats.get("options"), "options_distinct": stats.get("options_distinct"),
            "tokens_per_option": stats.get("tokens_per_option"), "states": states, "seq": seq, "room": room}


def main() -> None:
    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
    parser.add_argument("task", choices=tasks.TASKS)
    parser.add_argument("--base", default=None, help="a local copy of the base checkpoint (default: download it)")
    args = parser.parse_args()

    from transformers import AutoTokenizer

    from layatasks.train import base_dir

    task = tasks.load(args.task).TASK
    folder = data_dir(task.name)
    names = read_json(folder / "labels.json")["names"]
    tok = AutoTokenizer.from_pretrained(str(base_dir(args.base)) + "/tokenizer")
    train = check(task, tok, read_jsonl(folder / "train.jsonl"), names)
    test = check(task, tok, read_jsonl(folder / "test.jsonl"), names)
    padded = padded_tokens(train["seq"], task.epochs)
    result = {
        "task": task.name, "max_len": task.max_len, "head_max_len": task.head_max_len,
        "head_tokens": train["head_tokens"], "options": train["options"],
        "options_distinct": train["options_distinct"], "tokens_per_option": train["tokens_per_option"],
        "train": describe(train["states"], train["room"]),
        "test": dict(describe(test["states"], test["room"]),
                     truncated_rows=[i for i, n in enumerate(test["states"]) if n > test["room"]]),
        "estimate": {"epochs": task.epochs, "padded_tokens": padded, "reference": REFERENCE,
                     "minutes": (REFERENCE["seconds"] * padded / REFERENCE["padded_tokens"] / 60
                                 if REFERENCE["padded_tokens"] else None)},
    }
    write_json(folder / "fit.json", result)
    print("%s: head %d tokens for %d options (%s distinct); room for the text %d tokens"
          % (task.name, result["head_tokens"], result["options"], result["options_distinct"], train["room"]))
    for split in ("train", "test"):
        d = result[split]
        print("  %s: %d rows, text tokens median %s, p99 %d, max %d; %d cut short (%.1f%%)"
              % (split, d["rows"], d["state_tokens"]["median"], d["state_tokens"]["p99"], d["state_tokens"]["max"],
                 d["truncated"], 100 * d["truncated_share"]))
    if result["estimate"]["minutes"] is not None:
        print("  estimated fine-tune: %.0f minutes on the %s (scaled from %s, not measured)"
              % (result["estimate"]["minutes"], REFERENCE["gpu"], REFERENCE["name"]))


if __name__ == "__main__":
    main()