Update smoke_signal/scripts/03_ocr_bakeoff.py

smoke_signal/scripts/03_ocr_bakeoff.py

@@ -1,8 +1,20 @@
 #!/usr/bin/env python3
 """
-Smoke Signal - Stage 4: OCR Bake-Off
-Run Surya (primary) and Tesseract (fallback) on rendered pages.
-Freezes baseline OCR config after comparison.
+Smoke Signal — Stage 4: OCR Bake-Off
+======================================
+Runs Surya (primary) and Tesseract (fallback) on rendered page images
+from the calibration corpus. Compares outputs, measures accuracy against
+gold set if available, and freezes a baseline OCR config.
+
+Inputs:
+    renders/<BOOK_ID>/<BOOK_ID>_page_NNNN_300dpi.png
+    manifest/page_profiles/<BOOK_ID>_page_profile.json
+    manifest/source_manifest.csv
+
+Outputs:
+    ocr_raw/<BOOK_ID>/<BOOK_ID>_page_NNNN_ocr.json   — per-page OCR result
+    manifest/ocr_run_<RUN_ID>.json                    — run summary
+    configs/ss_ocr_config_v0.1.json                   — frozen baseline config
 
 Usage:
     python scripts/03_ocr_bakeoff.py
@@ -22,6 +34,7 @@ from datetime import datetime
 from pathlib import Path
 from typing import Optional
 
+# ── Paths ──────────────────────────────────────────────────────────────────────
 ROOT         = Path(__file__).resolve().parents[1]
 MANIFEST_CSV = ROOT / "manifest" / "source_manifest.csv"
 PROFILES_DIR = ROOT / "manifest" / "page_profiles"
@@ -30,9 +43,11 @@ OCR_RAW_DIR  = ROOT / "ocr_raw"
 CONFIGS_DIR  = ROOT / "configs"
 LOGS_DIR     = ROOT / "logs"
 
-for d in [OCR_RAW_DIR, CONFIGS_DIR, LOGS_DIR]:
-    d.mkdir(parents=True, exist_ok=True)
+OCR_RAW_DIR.mkdir(parents=True, exist_ok=True)
+CONFIGS_DIR.mkdir(parents=True, exist_ok=True)
+LOGS_DIR.mkdir(parents=True, exist_ok=True)
 
+# ── Frozen OCR config (do not change mid-batch) ────────────────────────────────
 OCR_CONFIG = {
     "config_version":      "ss_ocr_config_v0.1",
     "primary_engine":      "surya",
@@ -41,7 +56,7 @@ OCR_CONFIG = {
     "surya_det_batch":     4,
     "surya_rec_batch":     4,
     "tesseract_lang":      "eng",
-    "tesseract_psm":       6,
+    "tesseract_psm":       6,          # assume uniform block of text
     "confidence_threshold_auto_accept": 0.85,
     "confidence_threshold_review":      0.60,
     "confidence_threshold_quarantine":  0.40,
@@ -49,7 +64,8 @@ OCR_CONFIG = {
 }
 
 
-def load_manifest():
+# ── Manifest / profile loaders ─────────────────────────────────────────────────
+def load_manifest() -> dict:
     records = {}
     if not MANIFEST_CSV.exists():
         return records
@@ -60,187 +76,438 @@ def load_manifest():
     return records
 
 
-def save_manifest(records):
-    fields = ["book_id","source_id","filename","sha256","file_size_bytes",
-              "page_count","rights_class","source_location","acquisition_date",
-              "status","allowed_use","notes"]
-    rows = sorted(records.values(), key=lambda r: r.get("book_id",""))
+def save_manifest(records: dict) -> None:
+    fields = [
+        "book_id", "source_id", "filename", "sha256", "file_size_bytes",
+        "page_count", "rights_class", "source_location", "acquisition_date",
+        "status", "allowed_use", "notes"
+    ]
+    rows = sorted(records.values(), key=lambda r: r.get("book_id", ""))
     with open(MANIFEST_CSV, "w", newline="", encoding="utf-8") as f:
         writer = csv.DictWriter(f, fieldnames=fields)
         writer.writeheader()
         writer.writerows(rows)
 
 
-def load_page_profile(book_id):
-    p = PROFILES_DIR / f"{book_id}_page_profile.json"
-    if not p.exists():
+def load_page_profile(book_id: str) -> Optional[dict]:
+    profile_path = PROFILES_DIR / f"{book_id}_page_profile.json"
+    if not profile_path.exists():
         return None
-    with open(p, encoding="utf-8") as f:
+    with open(profile_path, encoding="utf-8") as f:
         return json.load(f)
 
 
-def run_surya(image_path, langs):
+# ── Surya OCR ──────────────────────────────────────────────────────────────────
+def _run_surya(image_path: Path, langs: list) -> dict:
+    """
+    Run Surya OCR on a single page image.
+    Returns standardised result dict.
+    """
     try:
         from PIL import Image
         from surya.ocr import run_ocr
-        from surya.model.detection.model import load_model as load_det
-        from surya.model.detection.processor import load_processor as load_det_proc
-        from surya.model.recognition.model import load_model as load_rec
-        from surya.model.recognition.processor import load_processor as load_rec_proc
+        from surya.model.detection.model import load_model as load_det_model
+        from surya.model.detection.processor import load_processor as load_det_processor
+        from surya.model.recognition.model import load_model as load_rec_model
+        from surya.model.recognition.processor import load_processor as load_rec_processor
     except ImportError as e:
-        return {"engine":"surya","error":str(e),"text":"","words":[],"confidence":0.0}
+        return {
+            "engine": "surya",
+            "error":  f"Import error: {e}. Run: pip install surya-ocr",
+            "text":   "",
+            "words":  [],
+            "confidence": 0.0,
+        }
+
     try:
-        img = Image.open(str(image_path)).convert("RGB")
-        det_m, det_p = load_det(), load_det_proc()
-        rec_m, rec_p = load_rec(), load_rec_proc()
-        results = run_ocr([img], [langs], det_m, det_p, rec_m, rec_p)
-        page = results[0]
-        words, confs, lines = [], [], []
-        for line in page.text_lines:
-            t = line.text.strip()
-            c = float(line.confidence) if hasattr(line,"confidence") else 1.0
-            if t:
-                lines.append(t)
-                confs.append(c)
-                words.append({"text":t,"confidence":round(c,4),"bbox":getattr(line,"bbox",None)})
-        avg = round(sum(confs)/len(confs),4) if confs else 0.0
-        return {"engine":"surya","text":"\n".join(lines),"words":words,"confidence":avg,"error":None}
+        image = Image.open(str(image_path)).convert("RGB")
+
+        det_model     = load_det_model()
+        det_processor = load_det_processor()
+        rec_model     = load_rec_model()
+        rec_processor = load_rec_processor()
+
+        results = run_ocr(
+            [image],
+            [langs],
+            det_model,
+            det_processor,
+            rec_model,
+            rec_processor,
+        )
+
+        page_result = results[0]
+
+        # Extract text and confidence from Surya's TextLine objects
+        words     = []
+        full_text = []
+        confidences = []
+
+        for line in page_result.text_lines:
+            text = line.text.strip()
+            conf = float(line.confidence) if hasattr(line, "confidence") else 1.0
+            if text:
+                full_text.append(text)
+                confidences.append(conf)
+                words.append({
+                    "text":       text,
+                    "confidence": round(conf, 4),
+                    "bbox":       line.bbox if hasattr(line, "bbox") else None,
+                })
+
+        avg_conf = round(sum(confidences) / len(confidences), 4) if confidences else 0.0
+
+        return {
+            "engine":     "surya",
+            "text":       "\n".join(full_text),
+            "words":      words,
+            "confidence": avg_conf,
+            "line_count": len(words),
+            "error":      None,
+        }
+
     except Exception as e:
-        return {"engine":"surya","error":str(e),"text":"","words":[],"confidence":0.0}
+        return {
+            "engine":     "surya",
+            "error":      str(e),
+            "text":       "",
+            "words":      [],
+            "confidence": 0.0,
+        }
 
 
-def run_tesseract(image_path, lang, psm):
+# ── Tesseract OCR (fallback) ───────────────────────────────────────────────────
+def _run_tesseract(image_path: Path, lang: str = "eng", psm: int = 6) -> dict:
+    """
+    Run Tesseract on a single page image.
+    Returns standardised result dict.
+    """
     try:
         import pytesseract
         from PIL import Image
     except ImportError as e:
-        return {"engine":"tesseract","error":str(e),"text":"","words":[],"confidence":0.0}
+        return {
+            "engine":     "tesseract",
+            "error":      f"Import error: {e}. Run: pip install pytesseract pillow",
+            "text":       "",
+            "words":      [],
+            "confidence": 0.0,
+        }
+
     try:
-        img = Image.open(str(image_path)).convert("RGB")
-        cfg = f"--psm {psm}"
-        data = pytesseract.image_to_data(img, lang=lang, config=cfg, output_type=pytesseract.Output.DICT)
-        words, confs = [], []
-        for i, t in enumerate(data["text"]):
-            t = str(t).strip()
-            c = int(data["conf"][i])
-            if t and c > 0:
-                cn = c/100.0
-                words.append({"text":t,"confidence":round(cn,4),"bbox":[data["left"][i],data["top"][i],data["left"][i]+data["width"][i],data["top"][i]+data["height"][i]]})
-                confs.append(cn)
-        avg = round(sum(confs)/len(confs),4) if confs else 0.0
-        text = pytesseract.image_to_string(img, lang=lang, config=cfg).strip()
-        return {"engine":"tesseract","text":text,"words":words,"confidence":avg,"error":None}
+        image = Image.open(str(image_path)).convert("RGB")
+        config = f"--psm {psm}"
+
+        # Get word-level data with confidence
+        data = pytesseract.image_to_data(
+            image,
+            lang=lang,
+            config=config,
+            output_type=pytesseract.Output.DICT,
+        )
+
+        words = []
+        confidences = []
+        full_text_parts = []
+
+        for i, word_text in enumerate(data["text"]):
+            word_text = str(word_text).strip()
+            conf = int(data["conf"][i])
+            if word_text and conf > 0:
+                conf_norm = conf / 100.0
+                words.append({
+                    "text":       word_text,
+                    "confidence": round(conf_norm, 4),
+                    "bbox": [
+                        data["left"][i], data["top"][i],
+                        data["left"][i] + data["width"][i],
+                        data["top"][i] + data["height"][i],
+                    ],
+                })
+                confidences.append(conf_norm)
+                full_text_parts.append(word_text)
+
+        avg_conf = round(sum(confidences) / len(confidences), 4) if confidences else 0.0
+        full_text = pytesseract.image_to_string(image, lang=lang, config=config).strip()
+
+        return {
+            "engine":     "tesseract",
+            "text":       full_text,
+            "words":      words,
+            "confidence": avg_conf,
+            "line_count": len(words),
+            "error":      None,
+        }
+
     except Exception as e:
-        return {"engine":"tesseract","error":str(e),"text":"","words":[],"confidence":0.0}
+        return {
+            "engine":     "tesseract",
+            "error":      str(e),
+            "text":       "",
+            "words":      [],
+            "confidence": 0.0,
+        }
+
 
+# ── Confidence gate ────────────────────────────────────────────────────────────
+def confidence_gate(confidence: float, config: dict) -> str:
+    """Return auto-accept | review-required | quarantine based on thresholds."""
+    if confidence >= config["confidence_threshold_auto_accept"]:
+        return "auto-accept"
+    elif confidence >= config["confidence_threshold_review"]:
+        return "review-required"
+    elif confidence >= config["confidence_threshold_quarantine"]:
+        return "quarantine"
+    else:
+        return "quarantine"
 
-def confidence_gate(conf, config):
-    if conf >= config["confidence_threshold_auto_accept"]: return "auto-accept"
-    if conf >= config["confidence_threshold_review"]:      return "review-required"
-    return "quarantine"
 
+# ── Per-page OCR ───────────────────────────────────────────────────────────────
+def ocr_page(
+    image_path: Path,
+    book_id: str,
+    page_num: int,
+    engine: str,
+    config: dict,
+    dry_run: bool = False,
+) -> dict:
+    """Run OCR on one page, save result, return summary."""
+
+    result = {
+        "book_id":    book_id,
+        "page_number": page_num,
+        "image_path": str(image_path),
+        "engine":     engine,
+        "ocr_at":     datetime.utcnow().isoformat() + "Z",
+        "config_version": config["config_version"],
+    }
 
-def ocr_page(image_path, book_id, page_num, engine, config, dry_run=False):
-    result = {"book_id":book_id,"page_number":page_num,"image_path":str(image_path),
-              "engine":engine,"ocr_at":datetime.utcnow().isoformat()+"Z",
-              "config_version":config["config_version"]}
     if dry_run:
-        result.update({"text":"[dry-run]","confidence":0.0,"gate":"dry-run","error":None,"words":[]})
+        result.update({
+            "text": "[dry-run]", "confidence": 0.0,
+            "gate": "dry-run", "error": None, "words": [],
+        })
         return result
-    ocr_out = run_surya(image_path, config["surya_langs"]) if engine == "surya" else \
-              run_tesseract(image_path, config["tesseract_lang"], config["tesseract_psm"])
+
+    if engine == "surya":
+        ocr_out = _run_surya(image_path, config["surya_langs"])
+    elif engine == "tesseract":
+        ocr_out = _run_tesseract(image_path, config["tesseract_lang"], config["tesseract_psm"])
+    else:
+        ocr_out = {"engine": engine, "error": f"Unknown engine: {engine}", "text": "", "words": [], "confidence": 0.0}
+
     result.update(ocr_out)
-    result["gate"] = confidence_gate(result.get("confidence",0.0), config)
-    out_dir = OCR_RAW_DIR / book_id
-    out_dir.mkdir(parents=True, exist_ok=True)
-    with open(out_dir / f"{book_id}_page_{page_num:04d}_{engine}_ocr.json","w",encoding="utf-8") as f:
+    result["gate"] = confidence_gate(result.get("confidence", 0.0), config)
+
+    # Save per-page OCR JSON
+    book_ocr_dir = OCR_RAW_DIR / book_id
+    book_ocr_dir.mkdir(parents=True, exist_ok=True)
+    out_path = book_ocr_dir / f"{book_id}_page_{page_num:04d}_{engine}_ocr.json"
+    with open(out_path, "w", encoding="utf-8") as f:
         json.dump(result, f, indent=2, ensure_ascii=False)
+
     return result
 
 
-def ocr_book(record, engine, dry_run=False):
-    book_id = record["book_id"]
-    print(f"\n  [{book_id}] {record['filename']} — {engine}")
+# ── Per-book OCR runner ──��─────────────────────────────────────────────────────
+def ocr_book(record: dict, engine: str, dry_run: bool = False) -> dict:
+    book_id  = record["book_id"]
+    print(f"\n  [{book_id}] {record['filename']} — engine: {engine}")
+
     profile = load_page_profile(book_id)
     if not profile:
-        print("    No profile. Run 02_profile_pdfs.py first.")
-        return {"book_id":book_id,"error":"no_profile","pages":[]}
-    eligible = [p for p in profile["pages"] if p.get("route") in OCR_CONFIG["eligible_routes"]]
-    print(f"    OCR pages: {len(eligible)} / {profile['page_count']}")
-    results, confs, gates, errors = [], [], {"auto-accept":0,"review-required":0,"quarantine":0,"dry-run":0}, []
-    for p in eligible:
-        pn = p["page_number"]
-        rp = p.get("render_path")
-        if not rp:
-            cands = list((RENDERS_DIR/book_id).glob(f"{book_id}_page_{pn:04d}_*.png")) if (RENDERS_DIR/book_id).exists() else []
-            rp = str(cands[0]) if cands else None
-        if not rp:
-            errors.append({"page":pn,"error":"no_render"}); continue
-        ip = Path(rp) if Path(rp).is_absolute() else ROOT/rp
-        if not ip.exists():
-            errors.append({"page":pn,"error":"render_missing"}); continue
-        r = ocr_page(ip, book_id, pn, engine, OCR_CONFIG, dry_run)
-        results.append(r)
-        c = r.get("confidence",0.0)
-        g = r.get("gate","quarantine")
-        confs.append(c)
-        gates[g] = gates.get(g,0)+1
-        sym = "v" if g=="auto-accept" else "!" if g=="review-required" else "x"
-        print(f"    {sym} p{pn:03d} conf={c:.2f} gate={g}")
-    avg = round(sum(confs)/len(confs),4) if confs else 0.0
-    print(f"    Avg={avg:.2f} | {gates}")
-    return {"book_id":book_id,"engine":engine,"pages_ocred":len(results),"avg_confidence":avg,"gate_counts":gates,"errors":errors,"error":None}
-
-
-def save_config(config):
-    p = CONFIGS_DIR / f"{config['config_version']}.json"
-    if not p.exists():
-        with open(p,"w",encoding="utf-8") as f:
-            json.dump({**config,"frozen_at":datetime.utcnow().isoformat()+"Z"},f,indent=2)
-        print(f"  Config frozen -> {p.relative_to(ROOT)}")
+        print(f"    ✗ No page profile found. Run 02_profile_pdfs.py first.")
+        return {"book_id": book_id, "error": "no_profile", "pages": []}
+
+    eligible_routes = OCR_CONFIG["eligible_routes"]
+    ocr_pages = [p for p in profile["pages"] if p.get("route") in eligible_routes]
+
+    print(f"    OCR-eligible pages: {len(ocr_pages)} / {profile['page_count']}")
+
+    if not ocr_pages:
+        print(f"    ✓ No OCR pages — all embedded text.")
+        return {"book_id": book_id, "error": None, "pages": [], "skipped": True}
+
+    page_results   = []
+    confidences    = []
+    gate_counts    = {"auto-accept": 0, "review-required": 0, "quarantine": 0, "dry-run": 0}
+    errors         = []
+
+    for page_info in ocr_pages:
+        page_num    = page_info["page_number"]
+        render_path = page_info.get("render_path")
 
+        if not render_path:
+            # Try to find render file
+            render_path_candidates = list((RENDERS_DIR / book_id).glob(
+                f"{book_id}_page_{page_num:04d}_*.png"
+            )) if (RENDERS_DIR / book_id).exists() else []
+            render_path = str(render_path_candidates[0]) if render_path_candidates else None
 
+        if not render_path:
+            print(f"    ⚠ Page {page_num}: no render found — skipping")
+            errors.append({"page": page_num, "error": "no_render"})
+            continue
+
+        image_path = Path(render_path) if Path(render_path).is_absolute() else ROOT / render_path
+
+        if not image_path.exists():
+            print(f"    ⚠ Page {page_num}: render file missing — {image_path}")
+            errors.append({"page": page_num, "error": "render_missing"})
+            continue
+
+        result = ocr_page(image_path, book_id, page_num, engine, OCR_CONFIG, dry_run)
+        page_results.append(result)
+
+        conf = result.get("confidence", 0.0)
+        gate = result.get("gate", "quarantine")
+        confidences.append(conf)
+        gate_counts[gate] = gate_counts.get(gate, 0) + 1
+
+        status = "✓" if gate == "auto-accept" else "⚠" if gate == "review-required" else "✗"
+        print(f"    {status} p{page_num:03d} conf={conf:.2f} gate={gate}")
+
+    avg_conf = round(sum(confidences) / len(confidences), 4) if confidences else 0.0
+
+    print(f"    Avg confidence : {avg_conf:.2f}")
+    print(f"    Gates          : {gate_counts}")
+    if errors:
+        print(f"    Errors         : {len(errors)}")
+
+    return {
+        "book_id":      book_id,
+        "engine":       engine,
+        "pages_ocred":  len(page_results),
+        "avg_confidence": avg_conf,
+        "gate_counts":  gate_counts,
+        "errors":       errors,
+        "error":        None,
+    }
+
+
+# ── Save frozen config ─────────────────────────────────────────────────────────
+def save_ocr_config(config: dict) -> None:
+    config_path = CONFIGS_DIR / f"{config['config_version']}.json"
+    if not config_path.exists():
+        with open(config_path, "w", encoding="utf-8") as f:
+            json.dump({
+                **config,
+                "frozen_at": datetime.utcnow().isoformat() + "Z",
+                "note": "DO NOT change this file mid-batch. Create a new version instead.",
+            }, f, indent=2)
+        print(f"\n  Config frozen → {config_path.relative_to(ROOT)}")
+    else:
+        print(f"\n  Config already exists → {config_path.relative_to(ROOT)} (not overwritten)")
+
+
+# ── Main ───────────────────────────────────────────────────────────────────────
 def main():
-    parser = argparse.ArgumentParser(description="Smoke Signal Stage 4: OCR Bake-Off")
-    parser.add_argument("--book-id")
-    parser.add_argument("--batch-id")
-    parser.add_argument("--engine", choices=["surya","tesseract","both"], default="surya")
-    parser.add_argument("--dry-run", action="store_true")
-    parser.add_argument("--all", action="store_true")
+    parser = argparse.ArgumentParser(description="Smoke Signal — Stage 4: OCR Bake-Off")
+    parser.add_argument("--book-id",  help="OCR a single book by ID")
+    parser.add_argument("--batch-id", help="Tag this run with a batch ID")
+    parser.add_argument("--engine",   choices=["surya", "tesseract", "both"],
+                        default="surya", help="OCR engine to use (default: surya)")
+    parser.add_argument("--dry-run",  action="store_true", help="No files written")
+    parser.add_argument("--all",      action="store_true", help="Include already-OCRed books")
     args = parser.parse_args()
+
     run_id  = args.batch_id or f"SS-RUN-{datetime.utcnow().strftime('%Y%m%d-%H%M%S')}"
-    engines = ["surya","tesseract"] if args.engine == "both" else [args.engine]
-    print(f"\nSmoke Signal Stage 4 | {run_id} | engines={engines}")
+    engines = ["surya", "tesseract"] if args.engine == "both" else [args.engine]
+
+    print(f"\n{'='*60}")
+    print(f"  Smoke Signal — Stage 4: OCR Bake-Off")
+    print(f"  Run ID  : {run_id}")
+    print(f"  Engines : {engines}")
+    print(f"  Config  : {OCR_CONFIG['config_version']}")
+    if args.dry_run:
+        print(f"  Mode    : DRY RUN")
+    print(f"{'='*60}")
+
     manifest = load_manifest()
     if not manifest:
-        print("Manifest empty. Run stage 1 first."); sys.exit(1)
-    eligible = ["profiled","rendered"] if not args.all else ["profiled","rendered","ocred"]
-    books = [manifest[args.book_id]] if args.book_id else [r for r in manifest.values() if r.get("status") in eligible]
+        print("\n  [error] Manifest empty. Run 01_register_sources.py first.")
+        sys.exit(1)
+
+    eligible_statuses = ["profiled", "rendered"] if not args.all else \
+                        ["profiled", "rendered", "ocred"]
+
+    if args.book_id:
+        books = [manifest[args.book_id]] if args.book_id in manifest else []
+        if not books:
+            print(f"  [error] Book {args.book_id} not in manifest.")
+            sys.exit(1)
+    else:
+        books = [r for r in manifest.values() if r.get("status") in eligible_statuses]
+
     if not books:
-        print(f"No books in status {eligible}. Run stage 3 first."); sys.exit(0)
-    print(f"Books: {len(books)}")
+        print(f"\n  No books with status in {eligible_statuses}.")
+        print("  Run 02_profile_pdfs.py first to render pages.")
+        sys.exit(0)
+
+    print(f"\n  Books to OCR: {len(books)}")
+
     all_results = []
-    t0 = time.time()
-    for rec in books:
-        for eng in engines:
-            res = ocr_book(rec, eng, args.dry_run)
-            all_results.append(res)
-            if not res.get("error") and not args.dry_run:
-                manifest[rec["book_id"]]["status"] = "ocred"
+    t_start     = time.time()
+
+    for record in books:
+        for engine in engines:
+            result = ocr_book(record, engine, dry_run=args.dry_run)
+            all_results.append(result)
+
+            # Update manifest status
+            if not result.get("error") and not args.dry_run:
+                manifest[record["book_id"]]["status"] = "ocred"
+
+    # Save manifest + config + run log
     if not args.dry_run:
         save_manifest(manifest)
-        save_config(OCR_CONFIG)
-        log = LOGS_DIR / f"{run_id}_ocr_bakeoff.json"
-        with open(log,"w") as f:
-            json.dump({"run_id":run_id,"engines":engines,"config":OCR_CONFIG,"results":all_results},f,indent=2)
-        print(f"  Log -> {log.relative_to(ROOT)}")
-    elapsed = round(time.time()-t0,1)
+        save_ocr_config(OCR_CONFIG)
+
+        log_path = LOGS_DIR / f"{run_id}_ocr_bakeoff.json"
+        with open(log_path, "w", encoding="utf-8") as f:
+            json.dump({
+                "run_id":   run_id,
+                "run_at":   datetime.utcnow().isoformat() + "Z",
+                "engines":  engines,
+                "config":   OCR_CONFIG,
+                "results":  all_results,
+            }, f, indent=2)
+        print(f"\n  Run log → {log_path.relative_to(ROOT)}")
+
+    # ── Summary ───────────────────────────────────────────────────────────────
+    elapsed   = round(time.time() - t_start, 1)
     succeeded = sum(1 for r in all_results if not r.get("error"))
-    total_pages = sum(r.get("pages_ocred",0) for r in all_results)
-    avg = sum(r.get("avg_confidence",0) for r in all_results if not r.get("error"))/max(succeeded,1)
-    print(f"\nDone: {succeeded}/{len(all_results)} books | {total_pages} pages | avg_conf={avg:.2f} | {elapsed}s")
-    print("Next: run 04_region_detector.py (Stage 5)")
+    total_pages = sum(r.get("pages_ocred", 0) for r in all_results)
+    avg_conf  = (
+        sum(r.get("avg_confidence", 0) for r in all_results if not r.get("error")) / max(succeeded, 1)
+    )
+
+    print(f"\n{'─'*60}")
+    print(f"  Books processed : {len(books)}")
+    print(f"  Runs succeeded  : {succeeded}")
+    print(f"  Pages OCR-ed    : {total_pages}")
+    print(f"  Avg confidence  : {avg_conf:.2f}")
+    print(f"  Time            : {elapsed}s")
+    print(f"{'─'*60}")
+
+    # Gate breakdown across all runs
+    total_gates = {"auto-accept": 0, "review-required": 0, "quarantine": 0}
+    for r in all_results:
+        for gate, count in r.get("gate_counts", {}).items():
+            if gate in total_gates:
+                total_gates[gate] += count
+
+    print(f"\n  Gate breakdown:")
+    for gate, count in total_gates.items():
+        pct = round(count / max(total_pages, 1) * 100, 1)
+        flag = "  ← ACTION REQUIRED" if gate != "auto-accept" and count > 0 else ""
+        print(f"    {gate:<20} {count:>4}  ({pct}%){flag}")
+
+    print(f"\n  Next steps:")
+    print(f"    1. Inspect ocr_raw/ outputs for quality")
+    print(f"    2. Run 04_region_detector.py (Stage 5)")
+    print(f"    3. Review quarantined pages manually\n")
 
 
 if __name__ == "__main__":
-    main()
+    main()