| """Build analysis/quant_report.html from analysis/quant_*.json files given on argv.""" |
| import sys, json, html, datetime |
| runs=[json.load(open(p)) for p in sys.argv[1:]] |
| ORDER=["fp32","fp32_onnx","int8_ort","int8_w","int4_w"] |
| LABEL={"fp32":"FP32 (PyTorch)","fp32_onnx":"FP32 (ONNX Runtime)","int8_ort":"INT8 static (ONNX Runtime)","int8_w":"INT8 weight-only (simulated)","int4_w":"INT4 weight-only (simulated)"} |
| def pct(v): return "" if v is None else f"{v*100:.1f}%" |
| def delta(v,ref): return "" if v is None else f"{(v-ref)*100:+.1f}" |
| def cls(v,ref): |
| if v is None: return "" |
| d=(v-ref)*100; return "good" if d>=-0.5 else ("warn" if d>=-3 else "bad") |
| css="""body{font-family:Segoe UI,system-ui,sans-serif;max-width:1100px;margin:2rem auto;padding:0 1rem;color:#222;line-height:1.45} |
| h1{font-size:1.6rem}h2{font-size:1.2rem;margin-top:2rem;border-bottom:1px solid #ddd;padding-bottom:.3rem} |
| table{border-collapse:collapse;width:100%;margin:1rem 0;font-size:.92rem}th,td{border:1px solid #ddd;padding:.45rem .6rem;text-align:right} |
| th:first-child,td:first-child{text-align:left}th{background:#f4f4f4}tr:nth-child(even){background:#fafafa} |
| .good{color:#1a7f37}.warn{color:#9a6700}.bad{color:#c0392b;font-weight:600}.muted{color:#666;font-size:.88rem} |
| .bar{display:inline-block;height:10px;background:#4a7bd0;vertical-align:middle;margin-right:.4rem}code{background:#f2f2f2;padding:0 .25rem} |
| .note{background:#fff8e1;border-left:4px solid #e0a800;padding:.6rem 1rem;margin:1rem 0}""" |
| out=[f"<!doctype html><html><head><meta charset='utf-8'><title>TinyFaceNet quantization report</title><style>{css}</style></head><body>", |
| f"<h1>TinyFaceNet quantization report</h1><p class='muted'>Generated {datetime.date.today()} · CPU i7-1360P · torch 2.12 / onnxruntime 1.26</p>", |
| "<div class='note'><b>How to read:</b> <i>Family acc</i> = centroid argmax on the date-held-out test faces (small set, so ±1 face ≈ ±2%). " |
| "<i>Open-set acc</i> = same but with unknown threshold 0.45 and 'other' included. " |
| "<i>Agreement</i> = share of 1500 faces from the full dataset where the quantized model picks the same person as FP32 — statistically the more reliable number. " |
| "<i>Cos</i> = mean cosine similarity of quantized vs FP32 embeddings. INT4 rows are weight-only simulations (CPU/Pi have no int4 conv kernels; the gain is file size only). " |
| "Latency here is noisy (measured while a training job was running) and is NOT Pi latency.</div>"] |
| for r in runs: |
| R=r["results"]; ref=R["fp32"] |
| out.append(f"<h2>{html.escape(r['tag'])} — <code>{html.escape(r['ckpt'])}</code></h2>") |
| out.append(f"<p class='muted'>test faces: {r['n_test']} ({r['n_test_family']} family) · agreement subset: {r['n_agree']} · THR={r['thr']}</p>") |
| out.append("<table><tr><th>Variant</th><th>Family acc</th><th>Δ pts</th><th>Open-set acc</th><th>Δ pts</th><th>Agreement vs FP32</th><th>Cos vs FP32</th><th>Size</th><th>Latency/face (this PC)</th><th>Note</th></tr>") |
| mx=max(R[k]["size_kb"] for k in ORDER if k in R) |
| for k in ORDER: |
| if k not in R: continue |
| v=R[k]; lat="—" if v["lat_ms"] is None else f"{v['lat_ms']:.1f} ms" |
| out.append(f"<tr><td>{LABEL[k]}</td><td>{pct(v['fam_acc'])}</td><td class='{cls(v['fam_acc'],ref['fam_acc'])}'>{delta(v['fam_acc'],ref['fam_acc'])}</td>" |
| f"<td>{pct(v['openset_acc'])}</td><td class='{cls(v['openset_acc'],ref['openset_acc'])}'>{delta(v['openset_acc'],ref['openset_acc'])}</td>" |
| f"<td class='{cls(v['argmax_agree_vs_fp32'],1.0)}'>{pct(v['argmax_agree_vs_fp32'])}</td><td>{v['cos_vs_fp32']:.4f}</td>" |
| f"<td><span class='bar' style='width:{int(120*v['size_kb']/mx)}px'></span>{v['size_kb']/1024:.2f} MB</td><td>{lat}</td><td class='muted'>{html.escape(v['note'])}</td></tr>") |
| out.append("</table>") |
| names=[n for n in r["names"] if n in ref["per_class"]] |
| out.append("<details><summary>Per-person accuracy (test set)</summary><table><tr><th>Variant</th>"+"".join(f"<th>{n}</th>" for n in names)+"</tr>") |
| for k in ORDER: |
| if k in R: out.append(f"<tr><td>{LABEL[k]}</td>"+"".join(f"<td>{pct(R[k]['per_class'].get(n))}</td>" for n in names)+"</tr>") |
| out.append("</table></details>") |
| out.append("<h2>Recommendation</h2><ul><li><b>Ship INT8 static (ONNX Runtime)</b>: ~1 MB, ~1 pt drop, runs natively on Pi 5 / Jetson with real int8 kernels.</li>" |
| "<li><b>INT4</b> buys only ~0.5 MB more and costs ~2-4 pts with no speed benefit on these targets. Not worth it for a 1M-param net.</li>" |
| "<li>If INT8 drop matters: quantization-aware fine-tune for 5 epochs, or calibrate with more images (200 used).</li></ul>") |
| out.append("</body></html>"); open("analysis/quant_report.html","w",encoding="utf-8").write("\n".join(out)); print("wrote analysis/quant_report.html") |
|
|
