scripts/eval.py

"""Eval: perplexity on held-out rows (best-results gate)."""
import json, math
from pathlib import Path
import torch

def main(path="data/processed/train.jsonl", holdout=200):
    rows = [json.loads(l) for l in open(path, encoding="utf-8")]
    test = rows[-holdout:]
    # char-level proxy PPL (real: use trained model loss)
    # proxy: uniform over 32000 vocab -> ppl ~32000; trained should be <<100
    n_chars = sum(len(r.get("answer", "")) for r in test)
    print(f"rows={len(rows)} holdout={len(test)} chars={n_chars}")
    print("proxy uniform PPL ~32000. After train.py, target PPL < 50 on this split.")
    print("Next: HumanEval pass@1 via bigcode-evaluation-harness (after real model).")

if __name__ == "__main__":
    main()