"""Experiment E: quantize the recognizer for CPU deployment (int8). Findings that shaped this tool (measured locally 2026-10-20, WSL CPU): * ORT *dynamic* quantization (`quantize_dynamic`, weights-only) makes this Conv+SE model SLOWER (0.69x fp32 throughput here: ConvInteger + per-activation quantize/dequantize dominates), although greedy ids are identical and max |dconf| is 0.077 on the smoke page. * => the deployment path is STATIC uint8 (QDQ) quantization with a small calibration set of real line crops; this script builds exactly that. The pipeline auto-loads `rec.int8.onnx` when it sits next to `rec.onnx` (indra_ocr/pipeline.py:195-200), so `--out models/indra-ocr-v3.1.1/rec.int8.onnx` is a drop-in. Acceptance (docs/INDRA_EXPERIMENTS.md, Experiment E): CER regression <= +0.3 pts absolute and >= 2x throughput — measure with `tools/exp/run_rec.py` on the frozen sets, both fp32 and int8. Calibration input: a manifest jsonl with the same shapes train_rec accepts {"image": "line.png", "text": "..."} (pre-cropped line image) {"image": "page.png", "box": [x0,y0,x1,y1], ...} (line box on a page) Paths relative to --calib-root (default: manifest's directory). Crops go through the SHARED rec_preproc.crop_line, so calibration sees exactly the inference distribution. 100-500 lines is plenty. Usage (remote): PYTHONPATH=.deps:$PWD/indra:$PWD/tools python3 tools/exp/e_int8_quantize.py \ --calib data/val_manifest.jsonl --calib-root ~/ocr/ocr-deva-ft/data \ --out models/indra-ocr-v3.1.1/rec.int8.onnx --check-smoke # then re-run Experiment A with the int8 model: python3 tools/exp/run_rec.py --onnx models/indra-ocr-v3.1.1/rec.int8.onnx ... """ from __future__ import annotations import argparse import json import sys from pathlib import Path import numpy as np sys.path.insert(0, str(Path(__file__).resolve().parent)) from common import REPO, load_gray, read_jsonl # noqa: E402 MODEL_DIR = REPO / "models" / "indra-ocr-v3.1.1" def build_calib(records: list[dict], root: Path, max_lines: int): """manifest -> list of [1,32,W] ink maps via the shared crop_line.""" from indra_ocr.rec_preproc import crop_line crops = [] for r in records: img = r.get("image") or r.get("file") p = Path(img) if not p.is_absolute(): p = (root / p).resolve() gray = load_gray(p) boxes = ([ln["box"] for ln in r["lines"]] if "lines" in r else [r.get("box")] if r.get("box") else [[0.0, 0.0, float(gray.shape[1]), float(gray.shape[0])]]) for b in boxes: crop = crop_line(gray, b) # [32,W] f32 crops.append(crop[None]) # [1,32,W] if len(crops) >= max_lines: return crops return crops class CalibReader: """Yields the recognizer's real inputs {x, widths} (batch = 1 line).""" def __init__(self, crops, multiple: int = 8): self.items = [] for c in crops: w = c.shape[2] wp = (w + multiple - 1) // multiple * multiple x = np.zeros((1, 1, 32, wp), np.float32) x[0, 0, :, :w] = c[0] self.items.append({"x": x, "widths": np.array([w], np.int64)}) self.i = 0 def get_next(self): if self.i >= len(self.items): return None it = self.items[self.i] self.i += 1 return it def rewind(self): self.i = 0 def smoke_check(fp_onnx: Path, int8_onnx: Path) -> dict: """ids / lengths / conf agreement on the recorded smoke page (indicative).""" import onnxruntime as ort from common import read_jsonl as _rj gt = _rj(REPO / "results" / "smoke" / "synth_nepali_gt.jsonl")[0] page = REPO / "results" / "smoke" / "synth_nepali_page.png" gray = load_gray(page) from indra_ocr.rec_preproc import crop_line, pad_batch crops = [crop_line(gray, ln["box"]) for ln in gt["lines"]] x, widths = pad_batch(crops) def run(path): sess = ort.InferenceSession(str(path), providers=["CPUExecutionProvider"]) return sess.run(None, {"x": x, "widths": widths}) f_ids, f_conf, f_len = run(fp_onnx) q_ids, q_conf, q_len = run(int8_onnx) id_bad = conf_delta = 0.0 id_bad = int(sum(int((f_ids[b, : f_len[b]] != q_ids[b, : q_len[b]]).sum()) for b in range(len(f_len)))) conf_delta = max(float(np.abs(f_conf[b, : f_len[b]] - q_conf[b, : q_len[b]]).max(initial=0.0)) for b in range(len(f_len))) return { "id_mismatches": id_bad, "lengths_equal": bool((f_len == q_len).all()), "max_abs_dconf": conf_delta, } def main(argv=None): ap = argparse.ArgumentParser("e_int8_quantize") ap.add_argument("--rec", type=Path, default=MODEL_DIR / "rec.onnx") ap.add_argument("--out", type=Path, default=MODEL_DIR / "rec.int8.onnx") ap.add_argument("--calib", type=Path, default=None, help="manifest jsonl of line crops/boxes (REQUIRED for static mode)") ap.add_argument("--calib-root", type=Path, default=None, help="root the manifest paths are relative to (default: manifest dir)") ap.add_argument("--max-lines", type=int, default=256) ap.add_argument("--method", choices=["static", "dynamic"], default="static") ap.add_argument("--per-channel", action="store_true", help="static: per-channel weight scales") ap.add_argument("--check-smoke", action="store_true", help="compare vs fp32 on the smoke page") args = ap.parse_args(argv) if not args.rec.exists(): raise SystemExit(f"--rec not found: {args.rec}") if args.method == "dynamic": from onnxruntime.quantization import QuantType, quantize_dynamic print("warning: dynamic (weights-only) quantization measured SLOWER than fp32 " "for this Conv+SE model — use static with calibration for deployment") quantize_dynamic(str(args.rec), str(args.out), weight_type=QuantType.QInt8, per_channel=args.per_channel) calib_n = 0 else: if not args.calib: raise SystemExit("static quantization needs --calib (manifest jsonl of line crops)") records = read_jsonl(args.calib) root = args.calib_root or args.calib.resolve().parent crops = build_calib(records, root, args.max_lines) if not crops: raise SystemExit(f"no crops built from {args.calib} — check paths/boxes") print(f"calibration: {len(crops)} line crops from {args.calib}") from onnxruntime.quantization import (CalibrationMethod, QuantFormat, QuantType, quantize_static) reader = CalibReader(crops) quantize_static( str(args.rec), str(args.out), reader, quant_format=QuantFormat.QDQ, activation_type=QuantType.QUInt8, weight_type=QuantType.QInt8, per_channel=args.per_channel, calibrate_method=CalibrationMethod.MinMax, ) calib_n = len(crops) report = { "method": args.method, "rec": str(args.rec), "out": str(args.out), "size_fp32_mb": round(args.rec.stat().st_size / 1e6, 2), "size_int8_mb": round(args.out.stat().st_size / 1e6, 2), "calib_lines": calib_n, } if args.check_smoke: report["smoke_vs_fp32"] = smoke_check(args.rec, args.out) report_path = args.out.with_suffix(".report.json") report_path.write_text(json.dumps(report, indent=1) + "\n") print(json.dumps(report, indent=1)) print(f"wrote {args.out} + {report_path}") print("NOTE: acceptance still requires CER on the frozen eval sets " "(run_rec.py fp32 vs int8) and throughput ≥2× on the deployment box.") if __name__ == "__main__": main()