| """Collect V4.2 Mac gate, task and measurement records into one acceptance status file.""" |
| import argparse |
| import json |
| import sys |
| from pathlib import Path |
|
|
| ROOT = Path(__file__).resolve().parents[2] |
| sys.path.insert(0, str(ROOT / "src")) |
| from vimeml.benchmarks.comparison import paired |
| from vimeml.training.data import file_sha |
|
|
| OUT = ROOT / "outputs/deployment" |
| TRACKS = ("standard-development", "legacy-regression", "ajimee", "historical-development") |
| EXPECTED_TOP1 = {"standard-development": 523, "legacy-regression": 370, "ajimee": 139, "historical-development": 123} |
| V21_TOP1 = {"standard-development": 505, "legacy-regression": 374, "ajimee": 144, "historical-development": 122} |
| RUNS = { |
| "cuda": ROOT / "outputs/evaluation-v42-20261020", |
| "mac_pytorch": OUT / "v42-mac-pytorch-baseline-v1/tracks", |
| "coreml_fp32": OUT / "v42-coreml-fp32-tracks-v1", |
| "coreml_int8": OUT / "v42-coreml-int8-tracks-v1", |
| "kv_fp32": OUT / "v42-coreml-kv-fp32-tracks-v1", |
| "kv_int8": OUT / "v42-coreml-kv-int8-tracks-v1", |
| "kvchunk_fp32": OUT / "v42-coreml-kvchunk-fp32-tracks-v1", |
| "kvchunk_int8": OUT / "v42-coreml-kvchunk-int8-tracks-v1", |
| "v21_coreml_fp32": OUT / "v42-v21-coreml-fp32-tracks-v1", |
| "v21_coreml_int8": OUT / "v42-v21-coreml-int8-tracks-v1", |
| } |
| METHODS = ("lm_context_sum", "lm_context_mean_secondary", "lm_no_context_sum") |
|
|
|
|
| def read(path): |
| return json.loads(Path(path).read_text(encoding="utf-8")) |
|
|
|
|
| def rows(run, track): |
| return [json.loads(line) for line in (RUNS[run] / track / "scores.jsonl").read_text(encoding="utf-8").splitlines() if line] |
|
|
|
|
| def all_method_orders(left, right): |
| """Per-case order differences for every LM method, not only the primary sum.""" |
| result = {} |
| for track in TRACKS: |
| a, b = rows(left, track), rows(right, track) |
| assert [r["id"] for r in a] == [r["id"] for r in b] |
| result[track] = {method: [x["id"] for x, y in zip(a, b) if x["orders"][method] != y["orders"][method]] |
| for method in METHODS} |
| return result |
|
|
|
|
| def top1(run): |
| out = {} |
| for track in TRACKS: |
| data = rows(run, track) |
| out[track] = sum(bool(r["orders"]["lm_context_sum"]) and r["orders"]["lm_context_sum"][0] in r["answers"] for r in data) |
| return out |
|
|
|
|
| def track_comparison(run): |
| summary = read(RUNS[run] / "summary.json") |
| return {track: summary["tracks"][track]["comparison"] for track in TRACKS} |
|
|
|
|
| def main(): |
| parser = argparse.ArgumentParser(description=__doc__) |
| parser.add_argument("--measurements", type=Path, required=True) |
| parser.add_argument("--resources", type=Path) |
| parser.add_argument("--output", type=Path, required=True) |
| args = parser.parse_args() |
| if args.output.exists(): |
| parser.error("Fresh output required.") |
| counts = {run: top1(run) for run in RUNS} |
| fixtures = read(OUT / "v42-mac-pytorch-baseline-v1/fixtures.json") |
| gates = {name: read(OUT / f"{name}/alignment.json") for name in ( |
| "v42-fp32-alignment-v1", "v42-kv-fp32-alignment-v1", "v42-int8-alignment-v1", |
| "v42-kv-int8-vs-stateless-int8-v1", "v42-kv-int8-vs-fp32-fixtures-v1", |
| "v42-kvchunk-fp32-alignment-v1", "v42-kvchunk-int8-vs-stateless-int8-v1", |
| "v42-kvchunk-int8-vs-fp32-fixtures-v1")} |
| orders = {f"{a}_vs_{b}": all_method_orders(a, b) for a, b in ( |
| ("cuda", "mac_pytorch"), ("mac_pytorch", "coreml_fp32"), ("mac_pytorch", "kv_fp32"), |
| ("coreml_fp32", "coreml_int8"), ("coreml_int8", "kv_int8"), |
| ("coreml_fp32", "kvchunk_fp32"), ("coreml_int8", "kvchunk_int8"))} |
| no_changes = lambda key: all(not ids for track in orders[key].values() for ids in track.values()) |
| int8 = track_comparison("coreml_int8") |
| measured = read(args.measurements) |
| checks = { |
| "1_mac_pytorch_fixtures_within_3e-4": fixtures["passed"], |
| "1_mac_pytorch_top1_equals_recorded_cuda": counts["mac_pytorch"] == EXPECTED_TOP1, |
| "1_mac_pytorch_orders_equal_cuda_all_methods": no_changes("cuda_vs_mac_pytorch"), |
| "2_fp32_stateless_and_kv_converted": True, |
| "3_fp32_stateless_alignment_passed": gates["v42-fp32-alignment-v1"]["passed"], |
| "3_fp32_kv_alignment_passed": gates["v42-kv-fp32-alignment-v1"]["passed"], |
| "3_coreml_fp32_top1_equals_mac_pytorch": counts["coreml_fp32"] == counts["mac_pytorch"], |
| "3_coreml_fp32_orders_equal_mac_pytorch_all_methods": no_changes("mac_pytorch_vs_coreml_fp32"), |
| "3_kv_fp32_top1_equals_mac_pytorch": counts["kv_fp32"] == counts["mac_pytorch"], |
| "3_kv_fp32_orders_equal_mac_pytorch_all_methods": no_changes("mac_pytorch_vs_kv_fp32"), |
| "4_int8_strict_logits_alignment_passed (expected false)": gates["v42-int8-alignment-v1"]["passed"], |
| "4_int8_kv_vs_same_weight_stateless_int8_passed": gates["v42-kv-int8-vs-stateless-int8-v1"]["passed"], |
| "4_kv_int8_top1_equals_stateless_int8": counts["kv_int8"] == counts["coreml_int8"], |
| "4_kv_int8_orders_equal_stateless_int8_all_methods": no_changes("coreml_int8_vs_kv_int8"), |
| "7_kvchunk_fp32_alignment_passed": gates["v42-kvchunk-fp32-alignment-v1"]["passed"], |
| "7_kvchunk_int8_vs_same_weight_stateless_int8_passed": gates["v42-kvchunk-int8-vs-stateless-int8-v1"]["passed"], |
| "7_kvchunk_int8_strict_vs_fp32_fixtures_passed (expected false)": gates["v42-kvchunk-int8-vs-fp32-fixtures-v1"]["passed"], |
| "7_kvchunk_fp32_top1_equals_stateless_fp32": counts["kvchunk_fp32"] == counts["coreml_fp32"], |
| "7_kvchunk_fp32_orders_equal_stateless_fp32_all_methods": no_changes("coreml_fp32_vs_kvchunk_fp32"), |
| "7_kvchunk_int8_top1_equals_stateless_int8": counts["kvchunk_int8"] == counts["coreml_int8"], |
| "7_kvchunk_int8_orders_equal_stateless_int8_all_methods": no_changes("coreml_int8_vs_kvchunk_int8"), |
| "8_v21_coreml_fp32_top1_equals_recorded": counts["v21_coreml_fp32"] == V21_TOP1, |
| "5_measurements_recorded": set(measured["models"]) >= {"v42-fp32", "v42-int8", "v21-fp32", "v21-int8"}, |
| "6_ios_resources_prepared": bool(args.resources and (args.resources / "manifest.json").is_file()), |
| } |
| status = {"format": "vimeml_v42_mac_acceptance_v1", "date": "2026-10-20", "checks": checks, |
| "top1": counts, "recorded_v21_fp32_top1": V21_TOP1, |
| "int8_vs_coreml_fp32": {track: {"fp32": int8[track]["reference_top1"], "int8": counts["coreml_int8"][track], |
| **int8[track]["paired"], "improved_ids": int8[track]["improved_ids"], "regressed_ids": int8[track]["regressed_ids"], |
| "order_changed_cases": int8[track]["order_changed_cases"], "top1_changed_cases": int8[track]["top1_changed_cases"], |
| "max_score_sum_abs": int8[track]["max_score_sum_abs"], "mean_score_sum_abs": int8[track]["mean_score_sum_abs"]} |
| for track in TRACKS}, |
| "v21_int8_vs_v42_int8": {track: {k: v for k, v in paired(rows("v21_coreml_int8", track), |
| rows("coreml_int8", track))["paired"].items() if k != "significant_at_0_05"} |
| for track in TRACKS}, |
| "v21_fp32_vs_v42_fp32": {track: {k: v for k, v in paired(rows("v21_coreml_fp32", track), |
| rows("coreml_fp32", track))["paired"].items() if k != "significant_at_0_05"} |
| for track in TRACKS}, |
| "v21_int8_vs_v21_fp32": {track: track_comparison("v21_coreml_int8")[track]["paired"] for track in TRACKS}, |
| "all_method_order_differences": orders, |
| "alignment_summaries": {name: {"passed": g["passed"], **g["summary"]} for name, g in gates.items()}, |
| "inputs_sha256": {"measurements": file_sha(args.measurements)}, |
| "policy": "Failed gates are results. No retraining, checkpoint selection, pool/reference change or scoring tuning."} |
| args.output.mkdir(parents=True) |
| (args.output / "status.json").write_text(json.dumps(status, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") |
| print(json.dumps({"checks": checks, "top1": counts}, ensure_ascii=False, indent=2)) |
|
|
|
|
| if __name__ == "__main__": |
| main() |
|
|
