#!/usr/bin/env python3 """Consolidate inspect + correctness + browser-latency JSONs into machine-readable tables. Reads the report JSONs produced by inspect_onnx.py, correctness.py and bench/bench_browser.mjs from a reports directory and emits: - combined.json (everything, keyed by model/variant) - sizes.csv, correctness.csv, latency.csv - markdown tables to stdout (paste-ready for the report) Usage: python aggregate_report.py --reports-dir DIR --variants-dir DIR [--out-dir DIR] """ from __future__ import annotations import argparse import csv import glob import json import os VARIANTS = ["fp32", "graphopt", "fp16", "int8-dynamic", "int8-static", "int8-static-selective"] MODELS = ["ecseg-s", "ecseg-m"] def load(path): with open(path) as fh: return json.load(fh) def size_bytes(variants_dir, model, variant): p = os.path.join(variants_dir, f"{model}.{variant}.onnx") return os.path.getsize(p) if os.path.exists(p) else None def collect_browser(reports_dir, engine="chrome"): """Merge browser results-*.json cells for one engine into {(model_file, config): result}. Keyed per engine so a WebKit run never overwrites the Chrome primary table; WebKit is reported separately in the report's cross-browser subsection. """ cells = {} host = None for path in sorted(glob.glob(os.path.join(reports_dir, "results-*.json"))): data = load(path) if data.get("host", {}).get("engine") != engine: continue host = data.get("host", host) for cell in data.get("cells", []): key = (cell.get("model"), cell.get("config")) cells[key] = cell return cells, host def md_table(headers, rows): out = ["| " + " | ".join(headers) + " |", "| " + " | ".join("---" for _ in headers) + " |"] for r in rows: out.append("| " + " | ".join(str(c) for c in r) + " |") return "\n".join(out) def fmt(x, nd=1): if x is None: return "—" if isinstance(x, float): return f"{x:.{nd}f}" return str(x) def main(): ap = argparse.ArgumentParser() ap.add_argument("--reports-dir", required=True) ap.add_argument("--variants-dir", required=True) ap.add_argument("--out-dir", default=None) args = ap.parse_args() out_dir = args.out_dir or args.reports_dir os.makedirs(out_dir, exist_ok=True) browser_cells, host = collect_browser(args.reports_dir) combined = {"host": host, "models": {}} # ---- sizes ---- size_rows = [] for model in MODELS: base = size_bytes(args.variants_dir, model, "fp32") for v in VARIANTS: b = size_bytes(args.variants_dir, model, v) pct = (100.0 * b / base) if (b and base) else None size_rows.append([model, v, b, fmt(b / 2**20, 2) if b else "—", fmt(pct, 1) if pct else "—"]) # ---- correctness ---- corr_rows = [] for model in MODELS: for v in VARIANTS: if v == "fp32": continue p = os.path.join(args.reports_dir, f"corr-{model}-{v}.json") if not os.path.exists(p): continue s = load(p)["summary"] corr_rows.append([ model, v, s.get("total_cand_instances"), s.get("total_base_instances"), fmt(s.get("mean_mask_iou_shared"), 4), fmt(s.get("worst_mask_iou_image_mean"), 3), fmt(s.get("mean_box_iou_shared"), 4), fmt(s.get("mean_mask_binary_flip_frac", 0) * 100, 2), s.get("near_conf_decision_flips"), str(s.get("any_nan_or_inf")), ]) # ---- latency ---- lat_rows = [] for model in MODELS: for v in VARIANTS: mf = f"{model}.{v}.onnx" for config in ("wasm-mt", "wasm-st"): cell = browser_cells.get((mf, config)) if not cell: continue if cell.get("ok") is False: lat_rows.append([model, v, config, "FAIL/STALL", "—", "—", "—", "—", cell.get("stalledAt") or cell.get("error", "")[:40]]) continue r = cell["result"] w = r["warm"] lat_rows.append([ model, v, config, fmt(r["sessionCreateMs"], 0), fmt(r["coldInferenceMs"], 0), fmt(w["p50"], 0), fmt(w["p90"], 0), fmt(w["p95"], 0), r["fingerprint"]["numInstances"], ]) # ---- write CSVs ---- def write_csv(name, header, rows): with open(os.path.join(out_dir, name), "w", newline="") as fh: w = csv.writer(fh) w.writerow(header) w.writerows(rows) write_csv("sizes.csv", ["model", "variant", "bytes", "MiB", "pct_of_fp32"], size_rows) write_csv("correctness.csv", ["model", "variant", "cand_inst", "base_inst", "mean_mask_iou", "worst_img_mask_iou", "mean_box_iou", "mask_flip_pct", "near_conf_flips", "nan_inf"], corr_rows) write_csv("latency.csv", ["model", "variant", "config", "create_ms", "cold_ms", "p50_ms", "p90_ms", "p95_ms", "instances"], lat_rows) combined["sizes"] = size_rows combined["correctness"] = corr_rows combined["latency"] = lat_rows with open(os.path.join(out_dir, "combined.json"), "w") as fh: json.dump(combined, fh, indent=2) # ---- markdown to stdout ---- print("### Sizes (bytes on disk = IndexedDB storage; storage is uncompressed)\n") print(md_table(["model", "variant", "MiB", "% of fp32"], [[r[0], r[1], r[3], r[4]] for r in size_rows])) print("\n### Correctness vs FP32 (held-out images)\n") print(md_table(["model", "variant", "cand/base inst", "mean maskIoU", "worst-img maskIoU", "mean boxIoU", "mask flip %", "near-conf flips", "NaN/Inf"], [[r[0], r[1], f"{r[2]}/{r[3]}", r[4], r[5], r[6], r[7], r[8], r[9]] for r in corr_rows])) print("\n### Browser latency (onnxruntime-web 1.24.3, CPU/WASM)\n") print(md_table(["model", "variant", "config", "create ms", "cold ms", "p50 ms", "p90 ms", "p95 ms", "inst"], lat_rows)) print(f"\nHost: {host}") print(f"\nWrote combined.json, sizes.csv, correctness.csv, latency.csv to {out_dir}") if __name__ == "__main__": main()