record_deltas.py

"""Measure the stock lm-eval HFLM delta vs run_lmeval on a SHARED subset.

Paired measurement: both paths run at the same --limit N; the reported delta
is hflm_subset - run_lmeval_subset (the pure EOS-prefix methodology delta).
The full-N published value is recorded as reference only. This is what the
model card's lm-eval honesty note is written from — measured, not asserted.
"""
import argparse
import json
import subprocess
import sys
from pathlib import Path

HERE = Path(__file__).resolve().parent
PY = sys.executable


def run_lmeval(tasks: list[str], limit: int) -> dict:
    out = subprocess.run(
        [str(PY), str(HERE / "run_lmeval.py"), "--effort", "medium",
         "--tasks", ",".join(tasks), "--json", "--limit", str(limit)],
        capture_output=True, text=True, cwd=str(HERE), timeout=3600,
    )
    if out.returncode != 0:
        raise RuntimeError(out.stderr[-2000:])
    return json.loads(out.stdout)


def run_hflm(tasks: list[str], limit: int) -> dict:
    from lm_eval import simple_evaluate
    out = simple_evaluate(
        model="hf",
        model_args=f"pretrained={HERE},trust_remote_code=True,dtype=float32,device=cpu",
        tasks=tasks,
        limit=limit,
    )
    return out["results"]


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--tasks", default="arc_easy,wikitext")
    ap.add_argument("--limit", type=int, default=200)
    args = ap.parse_args()
    tasks = args.tasks.split(",")

    repro = run_lmeval(tasks, args.limit)
    hflm = run_hflm(tasks, args.limit)

    deltas = {}
    for task in tasks:
        if task not in repro or task not in hflm:
            continue
        # Same metric key family per task (acc_norm for acc tasks, byte_ppl for wikitext)
        keys = [k for k in hflm[task] if k.endswith(("acc_norm,none", "acc,none", "byte_perplexity,none"))]
        if not keys:
            continue
        key = keys[0]
        deltas[task] = {
            "metric": key,
            "hflm_subset": hflm[task][key],
            "repro_subset": repro[task][key],
            "delta": hflm[task][key] - repro[task][key],  # the methodology delta
            "limit": args.limit,
        }
    (HERE / "deltas.json").write_text(json.dumps(deltas, indent=2))
    print(json.dumps(deltas, indent=2))


if __name__ == "__main__":
    main()