| """Pair frozen V1/V2 case outcomes and compute exact McNemar statistics.""" |
|
|
| import json |
| import sys |
| from pathlib import Path |
|
|
| ROOT = Path(__file__).resolve().parents[2] |
| sys.path.insert(0, str(ROOT / "src")) |
| from vimeml.training.data import write_json |
| from vimeml.benchmarks.comparison import paired, rows |
|
|
|
|
| def main(): |
| out = ROOT / "outputs/ime-eval/tiny-ja-v2.0-comparison" |
| out.mkdir(parents=True, exist_ok=True) |
| source = ROOT / "artifacts/models/tiny-ja-v2.0-e16k-d320-l6" |
| full = json.loads((source / "full-validation.json").read_text(encoding="utf-8")) |
| v1_bpc = 3.4736559 |
| result = { |
| "model_selection": "best.pt step 375000; minimum full BPC among last/subset-best and strongest dev result", |
| "v1_bpc": v1_bpc, |
| "v2_best_bpc": full["best"]["bpc"], |
| "v2_last_bpc": full["last"]["bpc"], |
| "relative_bpc_reduction": 1 - full["best"]["bpc"] / v1_bpc, |
| "benchmarks": {}, |
| "test_split_used": False, |
| "note": "Two small regression sets; larger frozen dev/blind sets are needed for robust selection.", |
| } |
| for name, v1_directory in [ |
| ("ajimee", "tiny-ja-v1-ajimee"), |
| ("development", "tiny-ja-v1-dev-v2-scores"), |
| ]: |
| before = rows(ROOT / "outputs/ime-eval" / v1_directory / "scores.jsonl") |
| best = rows(ROOT / "outputs/ime-eval" / f"tiny-ja-v2.0-best-{name}" / "scores.jsonl") |
| last = rows(source / "epoch-evaluation/epoch-4" / name / "scores.jsonl") |
| result["benchmarks"][name] = {"best": paired(before, best), "last": paired(before, last)} |
| write_json(out / "comparison.json", result) |
| print( |
| json.dumps( |
| {k: {m: x["paired"] for m, x in v.items()} for k, v in result["benchmarks"].items()}, |
| indent=2, |
| ) |
| ) |
|
|
|
|
| if __name__ == "__main__": |
| main() |
|
|
