"""Continue a text with the model: python generate.py "The problem of personal identity" Needs torch, safetensors and tokenizers. Sampling as on gustavecortal.com/philosophy-lm/: presence penalty 2.0, then temperature 0.7, top-k 20 and top-p 0.8. A text is continued as a new document (the end-of-text token first), as in training.""" import argparse import json import torch from safetensors.torch import load_file from tokenizers import Tokenizer from decode import Decoder from model import Transformer p = argparse.ArgumentParser() p.add_argument("text") p.add_argument("--tokens", type=int, default=256) p.add_argument("--temperature", type=float, default=0.7) p.add_argument("--top-k", type=int, default=20) p.add_argument("--top-p", type=float, default=0.8) p.add_argument("--presence-penalty", type=float, default=2.0) p.add_argument("--seed", type=int, default=0) args = p.parse_args() config = json.load(open("config.json")) model = Transformer(config["model"], config["vocab_size"]) model.load_state_dict(load_file("model.safetensors")) model.eval() tokenizer = Tokenizer.from_file("tokenizer.json") eos = config["eos_id"] torch.manual_seed(args.seed) decoder, seen, out = Decoder(model), set(), [] with torch.no_grad(): logits = decoder.prefill([eos] + tokenizer.encode(args.text, add_special_tokens=False).ids) for _ in range(args.tokens): logits = logits.float().clone() logits[list(seen)] -= args.presence_penalty top = torch.topk(logits / args.temperature, args.top_k) probs = torch.softmax(top.values, -1) keep = (torch.cumsum(probs, -1) - probs) < args.top_p token = top.indices[keep][torch.multinomial(probs[keep] / probs[keep].sum(), 1)].item() if token == eos: break seen.add(token) out.append(token) logits = decoder.step(token) print(args.text + tokenizer.decode(out))