File size: 4,653 Bytes
c83c617 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 | """Compare completed standard IME scores without rerunning model inference."""
import argparse
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT / "src"))
from vimeml.benchmarks.comparison import paired, rows
from vimeml.benchmarks.standard_ime import read
from vimeml.training.data import write_json
def comparison(before, after):
result = paired(before, after)
counts = result.pop("paired")
counts["before_wrong_after_correct"] = counts.pop("v1_wrong_v2_correct")
counts["before_correct_after_wrong"] = counts.pop("v1_correct_v2_wrong")
# Cases share authors/users; the unadjusted case-level p is exploratory.
counts.pop("significant_at_0_05")
result["before"] = result.pop("v1")
result["after"] = result.pop("v2")
result["paired"] = counts
result["top1_delta_percentage_points"] = 100 * (
result["after"]["top1_accuracy"] - result["before"]["top1_accuracy"])
return result
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--before", required=True, type=Path)
parser.add_argument("--after", required=True, type=Path)
parser.add_argument("--output", required=True, type=Path)
args = parser.parse_args()
if args.output.exists():
raise ValueError("Preserve prior comparisons; choose a new output.")
before_report = read(args.before / "metrics.json")
after_report = read(args.after / "metrics.json")
# Historical regression caches predate explicit label-quality metadata.
for report in (before_report, after_report):
if report.get("split") == "regression":
report.setdefault("label_quality", "legacy_historical_draft")
for key in ("format", "split", "label_quality", "labels_formal_gold", "policy", "result_role"):
if before_report[key] != after_report[key]:
raise ValueError(f"Evaluation protocol differs: {key}")
if before_report["format"] != "vimeml_standard_ime_metrics_v1":
raise ValueError("Expected completed standard IME reports.")
if before_report["status"] != "complete" or after_report["status"] != "complete":
raise ValueError("Both evaluations must be complete.")
if before_report["benchmark_manifest"] != after_report["benchmark_manifest"]:
raise ValueError("Frozen benchmark versions differ.")
if before_report["export_provenance"] != after_report["export_provenance"]:
raise ValueError("Converter/dictionary exports differ.")
if before_report["split"] == "blind":
if before_report["evaluation_plan"] != after_report["evaluation_plan"]:
raise ValueError("Blind comparisons must share the registered plan.")
before = rows(args.before / "scores.jsonl")
after = rows(args.after / "scores.jsonl")
after_by_id = {row["id"]: row for row in after}
for row in before:
other = after_by_id.get(row["id"])
if other is None or any(row[k] != other[k] for k in ("source", "provenance", "candidates")):
raise ValueError("Frozen source, provenance or candidate contents differ.")
sources = sorted({row["source"] for row in before})
report = {
"format": "vimeml_standard_ime_paired_v1", "status": "complete",
"split": before_report["split"], "result_role": before_report["result_role"],
"benchmark_release_id": before_report["benchmark_manifest"]["release_id"],
"label_quality": before_report["label_quality"],
"labels_formal_gold": before_report["labels_formal_gold"],
"before_model": before_report["model"], "after_model": after_report["model"],
"before_scores": str(args.before.resolve()), "after_scores": str(args.after.resolve()),
"policy": before_report["policy"],
"statistical_note": "Case-level exact McNemar p is exploratory: shared author/user groups and multiple comparisons are not adjusted; no equivalence claim.",
"by_source": {source: comparison([r for r in before if r["source"] == source],
[r for r in after if r["source"] == source]) for source in sources},
"micro": comparison(before, after),
}
if set(sources) == {"wrime", "jmultiwoz"}:
report["equal_source_macro_top1_delta_percentage_points"] = sum(
result["top1_delta_percentage_points"] for result in report["by_source"].values()) / 2
write_json(args.output, report)
print({"split": report["split"], "paired": report["micro"]["paired"],
"delta_pp": report["micro"]["top1_delta_percentage_points"]})
if __name__ == "__main__":
main()
|