Download OCR_INDRA/tools/exp/data_prep.py from prosperious/ocr-projects-archive: direct link, hf CLI and curl.
- Browser
- Download file 6.7 kB
-
https://huggingface.co/prosperious/ocr-projects-archive/resolve/main/OCR_INDRA/tools/exp/data_prep.py
- Command line
-
hf download hf://prosperious/ocr-projects-archive/OCR_INDRA/tools/exp/data_prep.py
-
curl -L -o data_prep.py https://huggingface.co/prosperious/ocr-projects-archive/resolve/main/OCR_INDRA/tools/exp/data_prep.py
6.7 kB
| """Build train/val manifests for Experiment B + leakage checks (plan §12). | |
| Format-agnostic entry points — the remote corpus layout is probe 5; these | |
| subcommands cover the layouts likely to exist, all emitting ONE format: | |
| {"image": ..., "box": [x0,y0,x1,y1]|null, "text": "..."} (jsonl) | |
| linepairs dir of line images + sibling transcripts (.txt files with the | |
| same stem, English names first-match) → manifest | |
| pages pages + per-line jsonl ({"image","lines":[{"box","text"}]}) → | |
| flat line manifest (boxes kept if present) | |
| split manifest → train/val jsonl by PAGE hash (0.9/0.1 default), so | |
| multi-line pages never straddle the split (§12.1 set purity) | |
| exclude remove val pages that sha256-match any page from the DEPLOYED | |
| pipeline's training renders (input: list of those image files) | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import hashlib | |
| import json | |
| import sys | |
| from pathlib import Path | |
| sys.path.insert(0, str(Path(__file__).resolve().parent)) | |
| from common import read_jsonl, write_jsonl # noqa: E402 | |
| def page_key(e: dict) -> str: | |
| return e.get("image", "") | |
| def _page_id(e: dict) -> tuple: | |
| """Identity of the PAGE an entry came from (same page ⇒ same split side).""" | |
| img = e.get("image", "") | |
| # pages are identified by their file; line-crop entries keep it too | |
| stem = Path(img).name.split("__")[0] | |
| return stem, img | |
| def _sha256(p: Path) -> str: | |
| h = hashlib.sha256() | |
| with open(p, "rb") as f: | |
| for chunk in iter(lambda: f.read(1 << 20), b""): | |
| h.update(chunk) | |
| return h.hexdigest() | |
| def cmd_linepairs(args) -> None: | |
| img_exts = {".png", ".jpg", ".jpeg", ".tif", ".tiff", ".bmp"} | |
| entries, missing_text, empty = [], 0, 0 | |
| for p in sorted(Path(args.dir).iterdir()): | |
| if p.suffix.lower() not in img_exts: | |
| continue | |
| txt_candidates = [p.with_suffix(".txt"), p.with_suffix(".TXT")] | |
| txt = next((t for t in txt_candidates), None) | |
| # also sibling dir variant: images/ + txt/ trees | |
| if txt is None or not txt.exists(): | |
| alt = args.dir.parent / "txt" / p.with_suffix(".txt").name | |
| if alt.exists(): | |
| txt = alt | |
| if txt is None or not txt.exists(): | |
| missing_text += 1 | |
| continue | |
| text = txt.read_text(encoding="utf-8", errors="strict").strip() | |
| if not text: | |
| empty += 1 | |
| continue | |
| entries.append({"image": str(p), "box": None, "text": text}) | |
| write_jsonl(args.out, entries) | |
| print(f"linepairs -> {args.out}: {len(entries)} lines " | |
| f"({missing_text} images without transcript skipped, {empty} empty texts)") | |
| def cmd_pages(args) -> None: | |
| entries = [] | |
| for r in read_jsonl(args.input): | |
| img = r.get("image") or r.get("file") | |
| for ln in r.get("lines", []): | |
| entries.append({ | |
| "image": img, | |
| "box": ln.get("box"), | |
| "text": ln["text"], | |
| }) | |
| write_jsonl(args.out, entries) | |
| print(f"pages -> {args.out}: {len(entries)} lines from {len(read_jsonl(args.input))} pages") | |
| def cmd_split(args) -> None: | |
| entries = read_jsonl(args.manifest) | |
| pages: dict[tuple, list[dict]] = {} | |
| for e in entries: | |
| pages.setdefault(_page_id(e), []).append(e) | |
| ids = sorted(pages) | |
| out_tr, out_va = [], [] | |
| for pid in ids: | |
| h = int(hashlib.sha256((":".join(map(str, pid))).encode()).hexdigest(), 16) / (1 << 128) | |
| side = out_va if h < args.val_frac else out_tr | |
| side.extend(pages[pid]) | |
| if not out_va and len(ids) > 1: | |
| # hash split can leave val empty on small corpora — move one whole page | |
| # over (page purity kept) so training always has a validation signal | |
| pid = min(ids, key=lambda k: hashlib.sha256((":".join(map(str, k))).encode()).hexdigest()) | |
| out_tr = [e for e in out_tr if _page_id(e) != pid] | |
| out_va = pages[pid] | |
| print(f"note: hash split left val empty; moved page {pid[0]!r} to val") | |
| write_jsonl(args.train_out, out_tr) | |
| write_jsonl(args.val_out, out_va) | |
| print(f"split -> {args.train_out} ({len(out_tr)} lines) / {args.val_out} ({len(out_va)} lines)" | |
| f" over {len(ids)} distinct pages") | |
| def cmd_exclude(args) -> None: | |
| """val pages matching (by content hash) ANY listed deployed-train page are dropped.""" | |
| train_hashes = set() | |
| listed = 0 | |
| for line in Path(args.deployed_pages).read_text().splitlines(): | |
| if not line.strip(): | |
| continue | |
| listed += 1 | |
| p = Path(line.strip()) | |
| if p.exists(): | |
| train_hashes.add(_sha256(p)) | |
| print(f"hashing deployed pages: {listed} listed, {len(train_hashes)} hashes read") | |
| dropped, kept = 0, 0 | |
| out = [] | |
| seen_pages: dict[tuple, str] = {} | |
| for e in read_jsonl(args.manifest): | |
| pid = _page_id(e) | |
| if pid not in seen_pages: | |
| src = Path(e["image"]) | |
| seen_pages[pid] = _sha256(src) if src.exists() else f"missing:{src}" | |
| if seen_pages[pid] in train_hashes: | |
| dropped += 1 | |
| else: | |
| out.append(e) | |
| kept = len(out) | |
| write_jsonl(args.out, out) | |
| print(f"exclude -> {args.out}: dropped {dropped} lines ({kept} kept); " | |
| f"train page hashes: {len(train_hashes)}") | |
| def main(argv=None): | |
| ap = argparse.ArgumentParser("data_prep") | |
| sub = ap.add_subparsers(dest="cmd", required=True) | |
| s = sub.add_parser("linepairs") | |
| s.add_argument("--dir", type=Path, required=True, help="dir with line images (+.txt)") | |
| s.add_argument("--out", type=Path, required=True) | |
| s.set_defaults(fn=cmd_linepairs) | |
| s = sub.add_parser("pages") | |
| s.add_argument("--input", type=Path, required=True, help="pages jsonl with box+text") | |
| s.add_argument("--out", type=Path, required=True) | |
| s.set_defaults(fn=cmd_pages) | |
| s = sub.add_parser("split") | |
| s.add_argument("--manifest", type=Path, required=True) | |
| s.add_argument("--train-out", type=Path, required=True) | |
| s.add_argument("--val-out", type=Path, required=True) | |
| s.add_argument("--val-frac", type=float, default=0.1) | |
| s.set_defaults(fn=cmd_split) | |
| s = sub.add_parser("exclude") | |
| s.add_argument("--manifest", type=Path, required=True, help="val manifest to filter") | |
| s.add_argument("--deployed-pages", type=Path, required=True, | |
| help="text file: one path per line of deployed TRAIN page images") | |
| s.add_argument("--out", type=Path, required=True) | |
| s.set_defaults(fn=cmd_exclude) | |
| args = ap.parse_args(argv) | |
| args.fn(args) | |
| if __name__ == "__main__": | |
| main() | |