PhiloLM / generate.py
gustavecortal's picture
philosophy-lm: model card, code, tokenizer and browser files
b9ee6b4 verified
Raw History Blame Contribute Delete
1.94 kB
"""Continue a text with the model: python generate.py "The problem of personal identity"
Needs torch, safetensors and tokenizers. Sampling as on gustavecortal.com/philosophy-lm/: presence penalty 2.0,
then temperature 0.7, top-k 20 and top-p 0.8. A text is continued as a new document (the end-of-text token first),
as in training."""
import argparse
import json
import torch
from safetensors.torch import load_file
from tokenizers import Tokenizer
from decode import Decoder
from model import Transformer
p = argparse.ArgumentParser()
p.add_argument("text")
p.add_argument("--tokens", type=int, default=256)
p.add_argument("--temperature", type=float, default=0.7)
p.add_argument("--top-k", type=int, default=20)
p.add_argument("--top-p", type=float, default=0.8)
p.add_argument("--presence-penalty", type=float, default=2.0)
p.add_argument("--seed", type=int, default=0)
args = p.parse_args()
config = json.load(open("config.json"))
model = Transformer(config["model"], config["vocab_size"])
model.load_state_dict(load_file("model.safetensors"))
model.eval()
tokenizer = Tokenizer.from_file("tokenizer.json")
eos = config["eos_id"]
torch.manual_seed(args.seed)
decoder, seen, out = Decoder(model), set(), []
with torch.no_grad():
logits = decoder.prefill([eos] + tokenizer.encode(args.text, add_special_tokens=False).ids)
for _ in range(args.tokens):
logits = logits.float().clone()
logits[list(seen)] -= args.presence_penalty
top = torch.topk(logits / args.temperature, args.top_k)
probs = torch.softmax(top.values, -1)
keep = (torch.cumsum(probs, -1) - probs) < args.top_p
token = top.indices[keep][torch.multinomial(probs[keep] / probs[keep].sum(), 1)].item()
if token == eos:
break
seen.add(token)
out.append(token)
logits = decoder.step(token)
print(args.text + tokenizer.decode(out))