Upload eval.py with huggingface_hub

eval.py

@@ -0,0 +1,95 @@
+import argparse
+import time
+from pathlib import Path
+
+import torch
+
+from indigo.common import (
+    build_tokenizer,
+    load_meta,
+    load_wordlist,
+    read_clean,
+    word_known_ratio,
+)
+from indigo.model import GPT, GPTConfig
+
+
+def muat(path):
+    from safetensors.torch import load_file
+
+    meta = load_meta(path)
+    model = GPT(GPTConfig(**meta["config"]))
+    missing, unexpected = model.load_state_dict(load_file(path), strict=False)
+    if missing or unexpected:
+        raise SystemExit(f"bobot tidak cocok untuk {path}: {missing[:3]} {unexpected[:3]}")
+    model.eval()
+    tokenizer = build_tokenizer(meta.get("tokenizer") or {"type": "char"}, meta.get("vocab"))
+    return model, tokenizer, meta
+
+
+@torch.no_grad()
+def nats_per_token(model, ids, block_size, device):
+    total_nll = 0.0
+    total_tok = 0
+    for i in range(0, max(0, len(ids) - 1), block_size):
+        potongan = ids[i : i + block_size + 1]
+        if len(potongan) < 2:
+            break
+        x = torch.tensor([potongan[:-1]], dtype=torch.long, device=device)
+        y = torch.tensor([potongan[1:]], dtype=torch.long, device=device)
+        logits, _ = model(x)
+        logp = torch.log_softmax(logits[0].float(), dim=-1)
+        total_nll += float(-logp[torch.arange(len(y[0])), y[0]].sum())
+        total_tok += len(potongan) - 1
+    return total_nll / max(1, total_tok), total_tok
+
+
+def main():
+    ap = argparse.ArgumentParser(
+        description="Skor checkpoint pada set uji tetap agar antar-run dapat dibandingkan"
+    )
+    ap.add_argument("--ckpt", nargs="+", required=True)
+    ap.add_argument("--test", default="data/sample.txt")
+    ap.add_argument("--device", default="cpu", choices=["cpu", "cuda"])
+    ap.add_argument("--guard", default=None, help="kamus opsional untuk metrik rasio ejaan")
+    ap.add_argument("--seed", type=int, default=42)
+    args = ap.parse_args()
+
+    teks = read_clean(args.test)
+    n_karakter = len(teks.encode("utf-8"))
+    wordset = prefiks = sufiks = None
+    if args.guard:
+        root = Path(__file__).resolve().parent
+        wordset = load_wordlist(args.guard)
+        p, s = root / "data" / "prefiks.txt", root / "data" / "sufiks.txt"
+        prefiks = load_wordlist(str(p)) if p.exists() else None
+        sufiks = load_wordlist(str(s)) if s.exists() else None
+
+    print(f"set uji: {args.test} ({n_karakter:,} karakter)")
+    print(f"{'checkpoint':44s} {'nats/tok':>9s} {'nat/kar':>8s} {'kamus':>7s}")
+    baris = []
+    for path in args.ckpt:
+        model, tokenizer, meta = muat(path)
+        ids = tokenizer.encode(teks)
+        npt, n_tok = nats_per_token(model, ids, meta["config"]["block_size"], args.device)
+        kompresi = n_karakter / max(1, n_tok)
+        npc = npt / kompresi
+        rasio = ""
+        if wordset:
+            t0 = time.time()
+            out = model.generate(
+                torch.tensor([[0]], dtype=torch.long, device=args.device),
+                120,
+                temperature=0.8,
+                top_k=40,
+            )
+            del t0
+            teks_out = tokenizer.decode(out[0].tolist())
+            rasio = f"{word_known_ratio(teks_out, wordset, prefiks, sufiks):6.0%}"
+        nama = Path(path).parent.parent.name + "/" + Path(path).name
+        print(f"{nama:44s} {npt:9.3f} {npc:8.3f} {rasio:>7s}")
+        baris.append({"ckpt": str(path), "nats_per_token": round(npt, 4), "nats_per_char": round(npc, 4)})
+
+
+if __name__ == "__main__":
+    main()