harness/dual_oracle.py

#!/usr/bin/env python3
"""Score a prediction file under BOTH oracles on the SAME benchmark.

PyLingual is OPTIONAL. Without it this still runs and still reports every number that depends
only on our oracle; the columns that need theirs are reported as null with an explicit
`pylingual_available: false`, never silently omitted and never quietly zeroed.

  ./dual_oracle.py --gen ../generations/gen_v3_csn.jsonl \\
                   --bench ../benchmarks/csn-3.12-licensed/bench.jsonl \\
                   --out ../results/dual_ours_csn.json --label "ours / CSN"
"""
from __future__ import annotations

import argparse
import json
import sys
import tempfile
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parent))

from common import (  # noqa: E402
    docstrings_of, load_bench, load_jsonl, ours_ok, pylingual_available, strip_fences, theirs_ok,
)


def main() -> None:
    ap = argparse.ArgumentParser()
    ap.add_argument("--gen", required=True)
    ap.add_argument("--bench", required=True)
    ap.add_argument("--out", required=True)
    ap.add_argument("--label", default="")
    a = ap.parse_args()

    bench, _ = load_bench(a.bench)
    gen = load_jsonl(a.gen)
    have_pyl = pylingual_available()
    if not have_pyl:
        print("NOTE: pylingual is not installed — their-oracle columns will be null. "
              "Our-oracle scoring is unaffected. See harness/README.md.", file=sys.stderr)

    n = ours = pyl = pyl_ok_ours_fail = 0
    doc_total = doc_recovered = 0
    with tempfile.TemporaryDirectory() as td:
        tmp = Path(td)
        for g in gen:
            i = g["i"]
            if i not in bench:
                continue
            n += 1
            ref = bench[i]
            src = strip_fences(g["got"])

            o_ok = ours_ok(src, ref["expected"])
            ours += o_ok

            if have_pyl:
                p_ok, _ = theirs_ok(src, Path(ref["pyc_path"]), tmp, f"g{i}")
                pyl += p_ok
                if p_ok and not o_ok:
                    pyl_ok_ours_fail += 1

            # docstring recovery, over references carrying a REAL docstring
            # (the CSN normal form rewrites docstrings to the literal 'pass' — not a real one)
            ref_docs = [d for d in docstrings_of(ref["expected"]) if d != "pass"]
            if ref_docs:
                doc_total += 1
                got = docstrings_of(src)
                if all(d in got for d in ref_docs):
                    doc_recovered += 1

    rep = {
        "label": a.label,
        "n": n,
        "pylingual_available": have_pyl,
        "PERFECT_our_oracle_docstring_strict": ours,
        "PERFECT_our_oracle_pct": round(100 * ours / max(1, n), 2),
        "PERFECT_their_oracle": pyl if have_pyl else None,
        "PERFECT_their_oracle_pct": round(100 * pyl / max(1, n), 2) if have_pyl else None,
        "passes_THEIRS_but_fails_OURS": pyl_ok_ours_fail if have_pyl else None,
        "samples_with_real_docstring": doc_total,
        "docstrings_exactly_recovered": doc_recovered,
        "docstring_recovery_pct": round(100 * doc_recovered / max(1, doc_total), 2),
    }
    Path(a.out).parent.mkdir(parents=True, exist_ok=True)
    Path(a.out).write_text(json.dumps(rep, indent=2))
    print(json.dumps(rep, indent=2))


if __name__ == "__main__":
    main()