File size: 1,897 Bytes
7b4e7cd | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 | """
Spark 2.0-code: пример запуска. python generate.py "Write a function that checks if a number is prime."
Нужна видеокарта NVIDIA (CUDA) и пакеты: torch, flash-linear-attention (fla), triton, tokenizers, safetensors.
Формат запроса такой же, как при обучении: ### User\n<задание>\n\n### Assistant\n
Генерация жадная (temperature 0): один и тот же запрос всегда даёт один и тот же ответ.
"""
import json, sys
from pathlib import Path
import torch
from safetensors.torch import load_file
from tokenizers import Tokenizer
from spark.model import HybridLM, ModelConfig
from spark.decode import Cache, GraphedStep, generate
HERE = Path(__file__).resolve().parent
TEMPLATE = "### User\n{content}\n\n### Assistant\n"
def load(device="cuda"):
cfg = ModelConfig(**json.loads((HERE / "config.json").read_text()))
model = HybridLM(cfg)
sd = load_file(str(HERE / "model.safetensors"))
sd["lm_head.weight"] = sd["embed.weight"] # веса общие (tie_embeddings)
model.load_state_dict(sd)
model = model.to(device, torch.bfloat16).eval()
tok = Tokenizer.from_file(str(HERE / "tokenizer.json"))
return model, tok
def ask(model, tok, task, max_new=1500):
eos = tok.token_to_id("<|endoftext|>")
cache = Cache(model, max_len=4096)
graphed = GraphedStep(model, cache)
ids = torch.tensor([tok.encode(TEMPLATE.format(content=task.strip())).ids], device="cuda")
out = []
with torch.no_grad():
generate(model, cache, graphed, ids, max_new, 0.0, 1.0, eos, 1.0, on_token=out.append)
return tok.decode(out)
if __name__ == "__main__":
task = " ".join(sys.argv[1:]) or "Write a Python function that checks whether a number is prime."
model, tok = load()
print(ask(model, tok, task))
|