File size: 1,897 Bytes
5f847d0
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
"""
Spark 2.0-code: пример запуска.   python generate.py "Write a function that checks if a number is prime."

Нужна видеокарта NVIDIA (CUDA) и пакеты: torch, flash-linear-attention (fla), triton, tokenizers, safetensors.
Формат запроса такой же, как при обучении:  ### User\n<задание>\n\n### Assistant\n
Генерация жадная (temperature 0): один и тот же запрос всегда даёт один и тот же ответ.
"""
import json, sys
from pathlib import Path

import torch
from safetensors.torch import load_file
from tokenizers import Tokenizer

from spark.model import HybridLM, ModelConfig
from spark.decode import Cache, GraphedStep, generate

HERE = Path(__file__).resolve().parent
TEMPLATE = "### User\n{content}\n\n### Assistant\n"


def load(device="cuda"):
    cfg = ModelConfig(**json.loads((HERE / "config.json").read_text()))
    model = HybridLM(cfg)
    sd = load_file(str(HERE / "model.safetensors"))
    sd["lm_head.weight"] = sd["embed.weight"]            # веса общие (tie_embeddings)
    model.load_state_dict(sd)
    model = model.to(device, torch.bfloat16).eval()
    tok = Tokenizer.from_file(str(HERE / "tokenizer.json"))
    return model, tok


def ask(model, tok, task, max_new=1500):
    eos = tok.token_to_id("<|endoftext|>")
    cache = Cache(model, max_len=4096)
    graphed = GraphedStep(model, cache)
    ids = torch.tensor([tok.encode(TEMPLATE.format(content=task.strip())).ids], device="cuda")
    out = []
    with torch.no_grad():
        generate(model, cache, graphed, ids, max_new, 0.0, 1.0, eos, 1.0, on_token=out.append)
    return tok.decode(out)


if __name__ == "__main__":
    task = " ".join(sys.argv[1:]) or "Write a Python function that checks whether a number is prime."
    model, tok = load()
    print(ask(model, tok, task))