"""Shared helpers for the experiment tools (matches tools/indra_eval/eval_indra.py conventions). Used by: train_rec.py, run_rec.py, exactness_check.py, adapters. Everything here is deliberately dependency-light: numpy + one of cv2 / PIL / pypdfium2. Imports the *cloned* `indra/` package (Apache-2.0) for Vocab + rec_preproc so training and inference run through one code path — the whole reason zero-shot and fine-tuned numbers are comparable (docs/INDRA_EXPERIMENTS.md §11). """ from __future__ import annotations import json import unicodedata from pathlib import Path REPO = Path(__file__).resolve().parents[2] for p in (str(REPO / "indra"),): if Path(p).is_dir(): import sys sys.path.insert(0, p) def load_gray(path: Path, dpi: float = 200.0): """Grayscale page as uint8 ndarray; PDF first page only (matches eval_indra).""" import numpy as np if path.suffix.lower() == ".pdf": import pypdfium2 as pdfium pdf = pdfium.PdfDocument(str(path)) img = pdf[0].render(scale=dpi / 72.0).to_pil() pdf.close() return np.asarray(img.convert("L")) import cv2 img = cv2.imread(str(path), cv2.IMREAD_GRAYSCALE) if img is None: from PIL import Image img = np.asarray(Image.open(path).convert("L")) return img def norm(s: str) -> str: return unicodedata.normalize("NFC", s) def read_jsonl(path: Path | str) -> list[dict]: path = Path(path) return [json.loads(ln) for ln in path.read_text(encoding="utf-8").splitlines() if ln.strip()] def write_jsonl(path: Path | str, records: list[dict]) -> None: path = Path(path) path.parent.mkdir(parents=True, exist_ok=True) path.write_text( "".join(json.dumps(r, ensure_ascii=False) + "\n" for r in records), encoding="utf-8", ) def append_jsonl(path: Path | str, record: dict) -> None: path = Path(path) path.parent.mkdir(parents=True, exist_ok=True) with open(path, "a", encoding="utf-8") as f: f.write(json.dumps(record, ensure_ascii=False) + "\n") def lev(a: str, b: str) -> int: """Codepoint Levenshtein, two-row DP (identical to eval_indra.py's scorer).""" if a == b: return 0 prev = list(range(len(b) + 1)) for i, ca in enumerate(a, 1): cur = [i] + [0] * len(b) for j, cb in enumerate(b, 1): cur[j] = min(prev[j] + 1, cur[j - 1] + 1, prev[j - 1] + (ca != cb)) prev = cur return prev[-1] def vocab(): """The recognizer's Vocab (1,677 tokens + blank 0). NFC normalize is inside encode().""" from indra_ocr.vocab import Vocab return Vocab.multi() def collapseIds(ids: list[int]) -> list[int]: """CTC greedy collapse: drop blank(0), skip repeats (pipeline.py:299 semantics).""" out: list[int] = [] prev = -1 for t in ids: if t != 0 and t != prev: out.append(int(t)) prev = t return out def mean_conf(pmax, lengths) -> list[tuple[float]]: """Per-line mean conf over VALID timesteps; None-safe for B=1 wrappers.""" mins = [float(pmax[k, : int(n)].mean()) if int(n) > 0 else 0.0 for k, n in enumerate(lengths)] return mins