File size: 6,460 Bytes
9496f98 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 | #!/usr/bin/env python3
"""Consolidate inspect + correctness + browser-latency JSONs into machine-readable tables.
Reads the report JSONs produced by inspect_onnx.py, correctness.py and bench/bench_browser.mjs
from a reports directory and emits:
- combined.json (everything, keyed by model/variant)
- sizes.csv, correctness.csv, latency.csv
- markdown tables to stdout (paste-ready for the report)
Usage:
python aggregate_report.py --reports-dir DIR --variants-dir DIR [--out-dir DIR]
"""
from __future__ import annotations
import argparse
import csv
import glob
import json
import os
VARIANTS = ["fp32", "graphopt", "fp16", "int8-dynamic", "int8-static", "int8-static-selective"]
MODELS = ["ecseg-s", "ecseg-m"]
def load(path):
with open(path) as fh:
return json.load(fh)
def size_bytes(variants_dir, model, variant):
p = os.path.join(variants_dir, f"{model}.{variant}.onnx")
return os.path.getsize(p) if os.path.exists(p) else None
def collect_browser(reports_dir, engine="chrome"):
"""Merge browser results-*.json cells for one engine into {(model_file, config): result}.
Keyed per engine so a WebKit run never overwrites the Chrome primary table; WebKit is reported
separately in the report's cross-browser subsection.
"""
cells = {}
host = None
for path in sorted(glob.glob(os.path.join(reports_dir, "results-*.json"))):
data = load(path)
if data.get("host", {}).get("engine") != engine:
continue
host = data.get("host", host)
for cell in data.get("cells", []):
key = (cell.get("model"), cell.get("config"))
cells[key] = cell
return cells, host
def md_table(headers, rows):
out = ["| " + " | ".join(headers) + " |", "| " + " | ".join("---" for _ in headers) + " |"]
for r in rows:
out.append("| " + " | ".join(str(c) for c in r) + " |")
return "\n".join(out)
def fmt(x, nd=1):
if x is None:
return "—"
if isinstance(x, float):
return f"{x:.{nd}f}"
return str(x)
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--reports-dir", required=True)
ap.add_argument("--variants-dir", required=True)
ap.add_argument("--out-dir", default=None)
args = ap.parse_args()
out_dir = args.out_dir or args.reports_dir
os.makedirs(out_dir, exist_ok=True)
browser_cells, host = collect_browser(args.reports_dir)
combined = {"host": host, "models": {}}
# ---- sizes ----
size_rows = []
for model in MODELS:
base = size_bytes(args.variants_dir, model, "fp32")
for v in VARIANTS:
b = size_bytes(args.variants_dir, model, v)
pct = (100.0 * b / base) if (b and base) else None
size_rows.append([model, v, b, fmt(b / 2**20, 2) if b else "—", fmt(pct, 1) if pct else "—"])
# ---- correctness ----
corr_rows = []
for model in MODELS:
for v in VARIANTS:
if v == "fp32":
continue
p = os.path.join(args.reports_dir, f"corr-{model}-{v}.json")
if not os.path.exists(p):
continue
s = load(p)["summary"]
corr_rows.append([
model, v,
s.get("total_cand_instances"), s.get("total_base_instances"),
fmt(s.get("mean_mask_iou_shared"), 4),
fmt(s.get("worst_mask_iou_image_mean"), 3),
fmt(s.get("mean_box_iou_shared"), 4),
fmt(s.get("mean_mask_binary_flip_frac", 0) * 100, 2),
s.get("near_conf_decision_flips"),
str(s.get("any_nan_or_inf")),
])
# ---- latency ----
lat_rows = []
for model in MODELS:
for v in VARIANTS:
mf = f"{model}.{v}.onnx"
for config in ("wasm-mt", "wasm-st"):
cell = browser_cells.get((mf, config))
if not cell:
continue
if cell.get("ok") is False:
lat_rows.append([model, v, config, "FAIL/STALL", "—", "—", "—", "—", cell.get("stalledAt") or cell.get("error", "")[:40]])
continue
r = cell["result"]
w = r["warm"]
lat_rows.append([
model, v, config,
fmt(r["sessionCreateMs"], 0),
fmt(r["coldInferenceMs"], 0),
fmt(w["p50"], 0), fmt(w["p90"], 0), fmt(w["p95"], 0),
r["fingerprint"]["numInstances"],
])
# ---- write CSVs ----
def write_csv(name, header, rows):
with open(os.path.join(out_dir, name), "w", newline="") as fh:
w = csv.writer(fh)
w.writerow(header)
w.writerows(rows)
write_csv("sizes.csv", ["model", "variant", "bytes", "MiB", "pct_of_fp32"], size_rows)
write_csv("correctness.csv",
["model", "variant", "cand_inst", "base_inst", "mean_mask_iou", "worst_img_mask_iou",
"mean_box_iou", "mask_flip_pct", "near_conf_flips", "nan_inf"], corr_rows)
write_csv("latency.csv",
["model", "variant", "config", "create_ms", "cold_ms", "p50_ms", "p90_ms", "p95_ms", "instances"], lat_rows)
combined["sizes"] = size_rows
combined["correctness"] = corr_rows
combined["latency"] = lat_rows
with open(os.path.join(out_dir, "combined.json"), "w") as fh:
json.dump(combined, fh, indent=2)
# ---- markdown to stdout ----
print("### Sizes (bytes on disk = IndexedDB storage; storage is uncompressed)\n")
print(md_table(["model", "variant", "MiB", "% of fp32"],
[[r[0], r[1], r[3], r[4]] for r in size_rows]))
print("\n### Correctness vs FP32 (held-out images)\n")
print(md_table(["model", "variant", "cand/base inst", "mean maskIoU", "worst-img maskIoU",
"mean boxIoU", "mask flip %", "near-conf flips", "NaN/Inf"],
[[r[0], r[1], f"{r[2]}/{r[3]}", r[4], r[5], r[6], r[7], r[8], r[9]] for r in corr_rows]))
print("\n### Browser latency (onnxruntime-web 1.24.3, CPU/WASM)\n")
print(md_table(["model", "variant", "config", "create ms", "cold ms", "p50 ms", "p90 ms", "p95 ms", "inst"],
lat_rows))
print(f"\nHost: {host}")
print(f"\nWrote combined.json, sizes.csv, correctness.csv, latency.csv to {out_dir}")
if __name__ == "__main__":
main()
|