prosperious's picture
Add 1000 files (OCR_INDRA/.gitignore ...)
f9bc0c1 verified
Raw History Blame Contribute Delete
3.19 kB
"""Shared helpers for the experiment tools (matches tools/indra_eval/eval_indra.py conventions).
Used by: train_rec.py, run_rec.py, exactness_check.py, adapters. Everything here
is deliberately dependency-light: numpy + one of cv2 / PIL / pypdfium2.
Imports the *cloned* `indra/` package (Apache-2.0) for Vocab + rec_preproc
so training and inference run through one code path — the whole reason
zero-shot and fine-tuned numbers are comparable (docs/INDRA_EXPERIMENTS.md §11).
"""
from __future__ import annotations
import json
import unicodedata
from pathlib import Path
REPO = Path(__file__).resolve().parents[2]
for p in (str(REPO / "indra"),):
if Path(p).is_dir():
import sys
sys.path.insert(0, p)
def load_gray(path: Path, dpi: float = 200.0):
"""Grayscale page as uint8 ndarray; PDF first page only (matches eval_indra)."""
import numpy as np
if path.suffix.lower() == ".pdf":
import pypdfium2 as pdfium
pdf = pdfium.PdfDocument(str(path))
img = pdf[0].render(scale=dpi / 72.0).to_pil()
pdf.close()
return np.asarray(img.convert("L"))
import cv2
img = cv2.imread(str(path), cv2.IMREAD_GRAYSCALE)
if img is None:
from PIL import Image
img = np.asarray(Image.open(path).convert("L"))
return img
def norm(s: str) -> str:
return unicodedata.normalize("NFC", s)
def read_jsonl(path: Path | str) -> list[dict]:
path = Path(path)
return [json.loads(ln) for ln in path.read_text(encoding="utf-8").splitlines() if ln.strip()]
def write_jsonl(path: Path | str, records: list[dict]) -> None:
path = Path(path)
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(
"".join(json.dumps(r, ensure_ascii=False) + "\n" for r in records),
encoding="utf-8",
)
def append_jsonl(path: Path | str, record: dict) -> None:
path = Path(path)
path.parent.mkdir(parents=True, exist_ok=True)
with open(path, "a", encoding="utf-8") as f:
f.write(json.dumps(record, ensure_ascii=False) + "\n")
def lev(a: str, b: str) -> int:
"""Codepoint Levenshtein, two-row DP (identical to eval_indra.py's scorer)."""
if a == b:
return 0
prev = list(range(len(b) + 1))
for i, ca in enumerate(a, 1):
cur = [i] + [0] * len(b)
for j, cb in enumerate(b, 1):
cur[j] = min(prev[j] + 1, cur[j - 1] + 1, prev[j - 1] + (ca != cb))
prev = cur
return prev[-1]
def vocab():
"""The recognizer's Vocab (1,677 tokens + blank 0). NFC normalize is inside encode()."""
from indra_ocr.vocab import Vocab
return Vocab.multi()
def collapseIds(ids: list[int]) -> list[int]:
"""CTC greedy collapse: drop blank(0), skip repeats (pipeline.py:299 semantics)."""
out: list[int] = []
prev = -1
for t in ids:
if t != 0 and t != prev:
out.append(int(t))
prev = t
return out
def mean_conf(pmax, lengths) -> list[tuple[float]]:
"""Per-line mean conf over VALID timesteps; None-safe for B=1 wrappers."""
mins = [float(pmax[k, : int(n)].mean()) if int(n) > 0 else 0.0 for k, n in enumerate(lengths)]
return mins