File size: 8,112 Bytes
779ff0b
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
"""Assemble the folder that is uploaded to Hugging Face: the weights, the exact question, the scores, the code,
the licences and the card.

Only what `laya.load` reads is copied from the run, plus what a user needs to ask the question the way the model was
trained, to check its numbers, and to credit its sources. The code folder carries the shared steps and this task's
module only.

The folder is refused when:

- it mentions a local path or the user name, because Laya's training config can record where the data lived;
- the card still holds a `{{...}}` slot, because the card's results are written by hand from the run's scores and a
  slot left in would publish a blank;
- the run's question is not the one the task defines, because the card would tell users to ask a question the model
  never saw.

Run it with `python -m layatasks.package <task> seed0 --repo <user>/laya-<task>`.
"""

from __future__ import annotations

import argparse
import hashlib
import re
import shutil
from pathlib import Path
from typing import List

from layatasks import task as tasks
from layatasks.common import data_dir, home, read_json, run_dir, write_json

REPO_ROOT = Path(__file__).resolve().parent.parent
MODEL_FILES = ("rl_agent_config.json", "model.safetensors", "questions.json")
MODEL_DIRS = ("tokenizer", "encoder")
CODE_FILES = ("README.md", "LICENSE", "requirements.txt", ".gitignore")
CORE_MODULES = ("__init__.py", "common.py", "compare.py", "data.py", "evaluate.py", "fit.py", "metrics.py",
                "package.py", "task.py", "train.py")
CORE_TESTS = ("__init__.py", "test_common.py", "test_data.py", "test_evaluate_gate.py", "test_fit.py",
              "test_metrics.py", "test_package.py", "test_train.py")
TEXT_SUFFIXES = (".json", ".md", ".txt", ".py", "")
SLOT = re.compile(r"\{\{[^}]*\}\}")


def placeholder(task) -> str:
    return "OWNER/" + task.release_name


def check_repo_id(repo: str, task) -> str:
    """A Hub model id, `owner/name`, with the characters the Hub allows."""
    if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*/[A-Za-z0-9][A-Za-z0-9_.-]*", repo) or repo == placeholder(task):
        raise ValueError("--repo must be a Hub id like someone/%s, got %r" % (task.release_name, repo))
    return repo


def looks_local(value: str) -> bool:
    """An absolute local path (`C:\\...`, `/home/...`, `~/...`, `\\\\server\\...`), not a Hub ID like `org/model`."""
    return bool(re.match(r"^([A-Za-z]:[\\/]|/|~|\\\\)", value))


def local_path_leaks(folder: Path, needles: List[str]) -> List[str]:
    """Files under `folder` that mention any of `needles`, compared case-insensitively and with either slash."""
    hits = []
    norm = [n.lower().replace("\\", "/") for n in needles if n]
    for path in folder.rglob("*"):
        # Configs, cards and code are small. The tokenizer's vocabulary is not, and short names can match a word piece.
        if not path.is_file() or path.suffix not in TEXT_SUFFIXES or path.stat().st_size > 1_000_000:
            continue
        text = path.read_text(encoding="utf-8", errors="ignore").lower().replace("\\\\", "/").replace("\\", "/")
        if any(n in text for n in norm):
            hits.append(str(path.relative_to(folder)))
    return hits


def scrub_config(path: Path) -> None:
    """Drop the training block's record of local file paths from the saved Laya config, keep everything else."""
    cfg = read_json(path)
    training = cfg.get("training")
    if isinstance(training, dict):
        for key in list(training):
            if isinstance(training[key], str) and looks_local(training[key]):
                training.pop(key)
    write_json(path, cfg)


def unfilled_slots(text: str) -> List[str]:
    return SLOT.findall(text)


def fill_repo(path: Path, old: str, repo: str) -> None:
    text = path.read_text(encoding="utf-8")
    path.write_text(text.replace(old, repo), encoding="utf-8", newline="\n")


def sha256sums(folder: Path) -> str:
    lines = []
    for path in sorted(p for p in folder.rglob("*") if p.is_file() and p.name != "SHA256SUMS"):
        h = hashlib.sha256()
        with open(path, "rb") as f:
            for block in iter(lambda: f.read(1 << 20), b""):
                h.update(block)
        lines.append("%s  %s" % (h.hexdigest(), path.relative_to(folder).as_posix()))
    return "\n".join(lines) + "\n"


def copy_scores(task, run: Path, out: Path) -> None:
    """The run's scores and training record, the data manifest and the fit report, without local paths."""
    result = read_json(run / "eval.json")
    result.pop("model", None)
    write_json(out / "eval.json", result)
    record = read_json(run / "train.json")
    record.get("summary", {}).pop("output_dir", None)
    write_json(out / "train.json", record)
    folder = data_dir(task.name)
    shutil.copy2(folder / "manifest.json", out / "data_manifest.json")
    if (folder / "fit.json").exists():
        shutil.copy2(folder / "fit.json", out / "fit.json")


def copy_code(task, out: Path) -> None:
    code = out / "code"
    (code / "layatasks" / "tasks").mkdir(parents=True)
    (code / "tests").mkdir()
    for name in CODE_FILES:
        shutil.copy2(REPO_ROOT / name, code / name)
    shutil.copy2(REPO_ROOT / "cards" / ("%s.NOTICE" % task.name), code / "NOTICE")
    for name in CORE_MODULES:
        shutil.copy2(REPO_ROOT / "layatasks" / name, code / "layatasks" / name)
    for name in ("__init__.py", "%s.py" % task.name):
        shutil.copy2(REPO_ROOT / "layatasks" / "tasks" / name, code / "layatasks" / "tasks" / name)
    for name in CORE_TESTS + ("test_task_%s.py" % task.name,):
        shutil.copy2(REPO_ROOT / "tests" / name, code / "tests" / name)


def build(name: str, run: str, repo: str) -> Path:
    task = tasks.load(name).TASK
    repo = check_repo_id(repo, task)
    card_src = REPO_ROOT / "cards" / ("%s.md" % task.name)
    slots = unfilled_slots(card_src.read_text(encoding="utf-8"))
    if slots:
        raise RuntimeError("cards/%s.md still has %d unfilled slots, first %s" % (task.name, len(slots), slots[0]))
    src_run = run_dir(task.name, run)
    src = src_run / "model"
    out = src_run / "release"
    if out.exists():
        shutil.rmtree(out)
    out.mkdir(parents=True)
    for f in MODEL_FILES:
        shutil.copy2(src / f, out / f)
    for d in MODEL_DIRS:
        shutil.copytree(src / d, out / d)
    scrub_config(out / "rl_agent_config.json")

    labels = read_json(data_dir(task.name) / "labels.json")
    if read_json(out / "questions.json") != {task.question_id: task.question(labels["names"])}:
        raise RuntimeError("the run's questions.json is not the question layatasks/tasks/%s.py defines" % task.name)
    write_json(out / "labels.json", {"source_label_names": labels["names"], "options": labels["options"]})

    copy_scores(task, src_run, out)
    shutil.copy2(REPO_ROOT / "LICENSE", out / "LICENSE")
    shutil.copy2(REPO_ROOT / "cards" / ("%s.NOTICE" % task.name), out / "NOTICE")
    shutil.copy2(card_src, out / "README.md")
    copy_code(task, out)
    for card in (out / "README.md", out / "code" / "README.md"):
        fill_repo(card, placeholder(task), repo)

    leaks = local_path_leaks(out, [str(Path.home()), Path.home().name, str(home())])
    if leaks:
        raise RuntimeError("these release files mention a local path or user name: %s" % ", ".join(leaks))
    left = [str(p.relative_to(out)) for p in out.rglob("*.md") if placeholder(task) in p.read_text(encoding="utf-8")]
    if left:
        raise RuntimeError("these files still name the placeholder repo: %s" % ", ".join(left))
    (out / "SHA256SUMS").write_text(sha256sums(out), encoding="utf-8", newline="\n")
    return out


def main() -> None:
    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
    parser.add_argument("task", choices=tasks.TASKS)
    parser.add_argument("run", help="the run to package, e.g. seed0")
    parser.add_argument("--repo", required=True, help="the Hugging Face model id it will be uploaded as")
    args = parser.parse_args()
    print(build(args.task, args.run, args.repo))


if __name__ == "__main__":
    main()