File size: 3,996 Bytes
0b19a1b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 | #!/usr/bin/env python3
"""Grade a prediction file under ONE oracle, with the harness self-tests run first.
./grade.py --bench <bench.jsonl> --gen <gen.jsonl> --out <report.json> # our oracle
./grade.py --bench <bench.jsonl> --gen <gen.jsonl> --oracle theirs --out ... # PyLingual's
./grade.py --bench <bench.jsonl> --self-test-only --out preflight.json
`--oracle ours` needs nothing but the Python standard library. `--oracle theirs` needs the
optional, user-installed PyLingual extra and exits with a clear message if it is absent.
Pre-flight and the mutation test run before any score is computed. If either is not 100% the
command REFUSES to print a score.
"""
from __future__ import annotations
import argparse
import json
import sys
import tempfile
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent))
from common import ( # noqa: E402
load_bench, load_jsonl, ours_ok, pylingual_available, require_sound, self_test, strip_fences,
theirs_ok,
)
def main() -> None:
ap = argparse.ArgumentParser()
ap.add_argument("--bench", required=True)
ap.add_argument("--gen")
ap.add_argument("--out", required=True)
ap.add_argument("--oracle", choices=("ours", "theirs"), default="ours")
ap.add_argument("--label", default="")
ap.add_argument("--self-test-only", action="store_true")
a = ap.parse_args()
if a.oracle == "theirs" and not pylingual_available():
raise SystemExit(
"--oracle theirs needs the optional PyLingual extra, which is not installed.\n"
"It is GPL-3.0 and is never vendored here; install it yourself (harness/README.md),\n"
"or use --oracle ours, which reproduces every number of ours without it."
)
bench, _ = load_bench(a.bench)
print(f"self-testing the {a.oracle} oracle on {len(bench)} labels...", file=sys.stderr, flush=True)
st = self_test(bench, a.oracle)
print(f" PRE-FLIGHT {st['preflight_perfect']}/{st['preflight_n']} = {st['preflight_pct']}% "
f"MUTATION {st['mutation_killed']}/{st['mutation_total']} killed = "
f"{st['mutation_kill_rate_pct']}%", file=sys.stderr, flush=True)
require_sound(st)
rep = {"label": a.label, "bench": str(a.bench), **st}
Path(a.out).parent.mkdir(parents=True, exist_ok=True)
if a.self_test_only or not a.gen:
Path(a.out).write_text(json.dumps(rep, indent=2))
print(json.dumps({k: v for k, v in rep.items()
if k not in ("preflight_failures", "mutation_survivors")}, indent=2))
return
gen = load_jsonl(a.gen)
n = perfect = 0
why_counts: dict[str, int] = {}
rows = []
with tempfile.TemporaryDirectory() as td:
tmp = Path(td)
for g in gen:
i = g["i"]
if i not in bench:
continue
n += 1
src = strip_fences(g["got"])
if a.oracle == "ours":
ok = ours_ok(src, bench[i]["expected"])
why = "PERFECT" if ok else "not byte-identical"
else:
ok, why = theirs_ok(src, Path(bench[i]["pyc_path"]), tmp, f"g{i}")
perfect += ok
if not ok:
key = why.split(":")[0]
why_counts[key] = why_counts.get(key, 0) + 1
rows.append({"i": i, "perfect": ok, "why": why,
"func": bench[i].get("csn_func", ""), "n_instr": bench[i].get("n_instr")})
rep.update({
"gen": str(a.gen),
"scored_n": n,
"PERFECT": perfect,
"PERFECT_pct": round(100 * perfect / max(1, n), 2),
"failure_profile": dict(sorted(why_counts.items(), key=lambda x: -x[1])),
})
Path(a.out).write_text(json.dumps({**rep, "rows": rows}, indent=2))
print(json.dumps({k: v for k, v in rep.items()
if k not in ("preflight_failures", "mutation_survivors")}, indent=2))
if __name__ == "__main__":
main()
|