Create eval/eval_perplexity.py

eval/eval_perplexity.py

@@ -0,0 +1,20 @@
+import math, json, torch
+from tokenizers import Tokenizer
+from model.tiny_gpt2 import TinyGPT2, GPTConfig
+
+tok = Tokenizer.from_file("out/tokenizer.json")
+cfg = GPTConfig(**json.load(open("out/pretrain/gpt_config.json")))
+m = TinyGPT2(cfg); m.load_state_dict(torch.load("out/sft/model_sft.pt", map_location="cpu")); m.eval()
+
+val_text = open("data/corpus_raw.txt","r",encoding="utf-8").read()[:20000]
+ids = tok.encode(val_text).ids
+losses = []
+with torch.no_grad():
+    for i in range(0, len(ids)-cfg.block_size, cfg.block_size):
+        x = torch.tensor([ids[i:i+cfg.block_size-1]], dtype=torch.long)
+        y = torch.tensor([ids[i+1:i+cfg.block_size]], dtype=torch.long)
+        logits = m(x)
+        loss = torch.nn.functional.cross_entropy(logits.view(-1, logits.size(-1)), y.view(-1))
+        losses.append(loss.item())
+ppl = math.exp(sum(losses)/len(losses)) if losses else float('inf')
+print(f"Perplexity ~ {ppl:.2f}")