| |
| """Consolidate inspect + correctness + browser-latency JSONs into machine-readable tables. |
| |
| Reads the report JSONs produced by inspect_onnx.py, correctness.py and bench/bench_browser.mjs |
| from a reports directory and emits: |
| - combined.json (everything, keyed by model/variant) |
| - sizes.csv, correctness.csv, latency.csv |
| - markdown tables to stdout (paste-ready for the report) |
| |
| Usage: |
| python aggregate_report.py --reports-dir DIR --variants-dir DIR [--out-dir DIR] |
| """ |
|
|
| from __future__ import annotations |
|
|
| import argparse |
| import csv |
| import glob |
| import json |
| import os |
|
|
| VARIANTS = ["fp32", "graphopt", "fp16", "int8-dynamic", "int8-static", "int8-static-selective"] |
| MODELS = ["ecseg-s", "ecseg-m"] |
|
|
|
|
| def load(path): |
| with open(path) as fh: |
| return json.load(fh) |
|
|
|
|
| def size_bytes(variants_dir, model, variant): |
| p = os.path.join(variants_dir, f"{model}.{variant}.onnx") |
| return os.path.getsize(p) if os.path.exists(p) else None |
|
|
|
|
| def collect_browser(reports_dir, engine="chrome"): |
| """Merge browser results-*.json cells for one engine into {(model_file, config): result}. |
| |
| Keyed per engine so a WebKit run never overwrites the Chrome primary table; WebKit is reported |
| separately in the report's cross-browser subsection. |
| """ |
| cells = {} |
| host = None |
| for path in sorted(glob.glob(os.path.join(reports_dir, "results-*.json"))): |
| data = load(path) |
| if data.get("host", {}).get("engine") != engine: |
| continue |
| host = data.get("host", host) |
| for cell in data.get("cells", []): |
| key = (cell.get("model"), cell.get("config")) |
| cells[key] = cell |
| return cells, host |
|
|
|
|
| def md_table(headers, rows): |
| out = ["| " + " | ".join(headers) + " |", "| " + " | ".join("---" for _ in headers) + " |"] |
| for r in rows: |
| out.append("| " + " | ".join(str(c) for c in r) + " |") |
| return "\n".join(out) |
|
|
|
|
| def fmt(x, nd=1): |
| if x is None: |
| return "—" |
| if isinstance(x, float): |
| return f"{x:.{nd}f}" |
| return str(x) |
|
|
|
|
| def main(): |
| ap = argparse.ArgumentParser() |
| ap.add_argument("--reports-dir", required=True) |
| ap.add_argument("--variants-dir", required=True) |
| ap.add_argument("--out-dir", default=None) |
| args = ap.parse_args() |
| out_dir = args.out_dir or args.reports_dir |
| os.makedirs(out_dir, exist_ok=True) |
|
|
| browser_cells, host = collect_browser(args.reports_dir) |
| combined = {"host": host, "models": {}} |
|
|
| |
| size_rows = [] |
| for model in MODELS: |
| base = size_bytes(args.variants_dir, model, "fp32") |
| for v in VARIANTS: |
| b = size_bytes(args.variants_dir, model, v) |
| pct = (100.0 * b / base) if (b and base) else None |
| size_rows.append([model, v, b, fmt(b / 2**20, 2) if b else "—", fmt(pct, 1) if pct else "—"]) |
|
|
| |
| corr_rows = [] |
| for model in MODELS: |
| for v in VARIANTS: |
| if v == "fp32": |
| continue |
| p = os.path.join(args.reports_dir, f"corr-{model}-{v}.json") |
| if not os.path.exists(p): |
| continue |
| s = load(p)["summary"] |
| corr_rows.append([ |
| model, v, |
| s.get("total_cand_instances"), s.get("total_base_instances"), |
| fmt(s.get("mean_mask_iou_shared"), 4), |
| fmt(s.get("worst_mask_iou_image_mean"), 3), |
| fmt(s.get("mean_box_iou_shared"), 4), |
| fmt(s.get("mean_mask_binary_flip_frac", 0) * 100, 2), |
| s.get("near_conf_decision_flips"), |
| str(s.get("any_nan_or_inf")), |
| ]) |
|
|
| |
| lat_rows = [] |
| for model in MODELS: |
| for v in VARIANTS: |
| mf = f"{model}.{v}.onnx" |
| for config in ("wasm-mt", "wasm-st"): |
| cell = browser_cells.get((mf, config)) |
| if not cell: |
| continue |
| if cell.get("ok") is False: |
| lat_rows.append([model, v, config, "FAIL/STALL", "—", "—", "—", "—", cell.get("stalledAt") or cell.get("error", "")[:40]]) |
| continue |
| r = cell["result"] |
| w = r["warm"] |
| lat_rows.append([ |
| model, v, config, |
| fmt(r["sessionCreateMs"], 0), |
| fmt(r["coldInferenceMs"], 0), |
| fmt(w["p50"], 0), fmt(w["p90"], 0), fmt(w["p95"], 0), |
| r["fingerprint"]["numInstances"], |
| ]) |
|
|
| |
| def write_csv(name, header, rows): |
| with open(os.path.join(out_dir, name), "w", newline="") as fh: |
| w = csv.writer(fh) |
| w.writerow(header) |
| w.writerows(rows) |
|
|
| write_csv("sizes.csv", ["model", "variant", "bytes", "MiB", "pct_of_fp32"], size_rows) |
| write_csv("correctness.csv", |
| ["model", "variant", "cand_inst", "base_inst", "mean_mask_iou", "worst_img_mask_iou", |
| "mean_box_iou", "mask_flip_pct", "near_conf_flips", "nan_inf"], corr_rows) |
| write_csv("latency.csv", |
| ["model", "variant", "config", "create_ms", "cold_ms", "p50_ms", "p90_ms", "p95_ms", "instances"], lat_rows) |
|
|
| combined["sizes"] = size_rows |
| combined["correctness"] = corr_rows |
| combined["latency"] = lat_rows |
| with open(os.path.join(out_dir, "combined.json"), "w") as fh: |
| json.dump(combined, fh, indent=2) |
|
|
| |
| print("### Sizes (bytes on disk = IndexedDB storage; storage is uncompressed)\n") |
| print(md_table(["model", "variant", "MiB", "% of fp32"], |
| [[r[0], r[1], r[3], r[4]] for r in size_rows])) |
| print("\n### Correctness vs FP32 (held-out images)\n") |
| print(md_table(["model", "variant", "cand/base inst", "mean maskIoU", "worst-img maskIoU", |
| "mean boxIoU", "mask flip %", "near-conf flips", "NaN/Inf"], |
| [[r[0], r[1], f"{r[2]}/{r[3]}", r[4], r[5], r[6], r[7], r[8], r[9]] for r in corr_rows])) |
| print("\n### Browser latency (onnxruntime-web 1.24.3, CPU/WASM)\n") |
| print(md_table(["model", "variant", "config", "create ms", "cold ms", "p50 ms", "p90 ms", "p95 ms", "inst"], |
| lat_rows)) |
| print(f"\nHost: {host}") |
| print(f"\nWrote combined.json, sizes.csv, correctness.csv, latency.csv to {out_dir}") |
|
|
|
|
| if __name__ == "__main__": |
| main() |
|
|
