Download OCR_INDRA/tools/exp/common.py from prosperious/ocr-projects-archive: direct link, hf CLI and curl.
- Browser
- Download file 3.19 kB
-
https://huggingface.co/prosperious/ocr-projects-archive/resolve/main/OCR_INDRA/tools/exp/common.py
- Command line
-
hf download hf://prosperious/ocr-projects-archive/OCR_INDRA/tools/exp/common.py
-
curl -L -o common.py https://huggingface.co/prosperious/ocr-projects-archive/resolve/main/OCR_INDRA/tools/exp/common.py
3.19 kB
| """Shared helpers for the experiment tools (matches tools/indra_eval/eval_indra.py conventions). | |
| Used by: train_rec.py, run_rec.py, exactness_check.py, adapters. Everything here | |
| is deliberately dependency-light: numpy + one of cv2 / PIL / pypdfium2. | |
| Imports the *cloned* `indra/` package (Apache-2.0) for Vocab + rec_preproc | |
| so training and inference run through one code path — the whole reason | |
| zero-shot and fine-tuned numbers are comparable (docs/INDRA_EXPERIMENTS.md §11). | |
| """ | |
| from __future__ import annotations | |
| import json | |
| import unicodedata | |
| from pathlib import Path | |
| REPO = Path(__file__).resolve().parents[2] | |
| for p in (str(REPO / "indra"),): | |
| if Path(p).is_dir(): | |
| import sys | |
| sys.path.insert(0, p) | |
| def load_gray(path: Path, dpi: float = 200.0): | |
| """Grayscale page as uint8 ndarray; PDF first page only (matches eval_indra).""" | |
| import numpy as np | |
| if path.suffix.lower() == ".pdf": | |
| import pypdfium2 as pdfium | |
| pdf = pdfium.PdfDocument(str(path)) | |
| img = pdf[0].render(scale=dpi / 72.0).to_pil() | |
| pdf.close() | |
| return np.asarray(img.convert("L")) | |
| import cv2 | |
| img = cv2.imread(str(path), cv2.IMREAD_GRAYSCALE) | |
| if img is None: | |
| from PIL import Image | |
| img = np.asarray(Image.open(path).convert("L")) | |
| return img | |
| def norm(s: str) -> str: | |
| return unicodedata.normalize("NFC", s) | |
| def read_jsonl(path: Path | str) -> list[dict]: | |
| path = Path(path) | |
| return [json.loads(ln) for ln in path.read_text(encoding="utf-8").splitlines() if ln.strip()] | |
| def write_jsonl(path: Path | str, records: list[dict]) -> None: | |
| path = Path(path) | |
| path.parent.mkdir(parents=True, exist_ok=True) | |
| path.write_text( | |
| "".join(json.dumps(r, ensure_ascii=False) + "\n" for r in records), | |
| encoding="utf-8", | |
| ) | |
| def append_jsonl(path: Path | str, record: dict) -> None: | |
| path = Path(path) | |
| path.parent.mkdir(parents=True, exist_ok=True) | |
| with open(path, "a", encoding="utf-8") as f: | |
| f.write(json.dumps(record, ensure_ascii=False) + "\n") | |
| def lev(a: str, b: str) -> int: | |
| """Codepoint Levenshtein, two-row DP (identical to eval_indra.py's scorer).""" | |
| if a == b: | |
| return 0 | |
| prev = list(range(len(b) + 1)) | |
| for i, ca in enumerate(a, 1): | |
| cur = [i] + [0] * len(b) | |
| for j, cb in enumerate(b, 1): | |
| cur[j] = min(prev[j] + 1, cur[j - 1] + 1, prev[j - 1] + (ca != cb)) | |
| prev = cur | |
| return prev[-1] | |
| def vocab(): | |
| """The recognizer's Vocab (1,677 tokens + blank 0). NFC normalize is inside encode().""" | |
| from indra_ocr.vocab import Vocab | |
| return Vocab.multi() | |
| def collapseIds(ids: list[int]) -> list[int]: | |
| """CTC greedy collapse: drop blank(0), skip repeats (pipeline.py:299 semantics).""" | |
| out: list[int] = [] | |
| prev = -1 | |
| for t in ids: | |
| if t != 0 and t != prev: | |
| out.append(int(t)) | |
| prev = t | |
| return out | |
| def mean_conf(pmax, lengths) -> list[tuple[float]]: | |
| """Per-line mean conf over VALID timesteps; None-safe for B=1 wrappers.""" | |
| mins = [float(pmax[k, : int(n)].mean()) if int(n) > 0 else 0.0 for k, n in enumerate(lengths)] | |
| return mins | |