prosperious's picture
Add 1000 files (OCR_INDRA/.gitignore ...)
f9bc0c1 verified
Raw History Blame Contribute Delete
6.7 kB
"""Build train/val manifests for Experiment B + leakage checks (plan §12).
Format-agnostic entry points — the remote corpus layout is probe 5; these
subcommands cover the layouts likely to exist, all emitting ONE format:
{"image": ..., "box": [x0,y0,x1,y1]|null, "text": "..."} (jsonl)
linepairs dir of line images + sibling transcripts (.txt files with the
same stem, English names first-match) → manifest
pages pages + per-line jsonl ({"image","lines":[{"box","text"}]}) →
flat line manifest (boxes kept if present)
split manifest → train/val jsonl by PAGE hash (0.9/0.1 default), so
multi-line pages never straddle the split (§12.1 set purity)
exclude remove val pages that sha256-match any page from the DEPLOYED
pipeline's training renders (input: list of those image files)
"""
from __future__ import annotations
import argparse
import hashlib
import json
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent))
from common import read_jsonl, write_jsonl # noqa: E402
def page_key(e: dict) -> str:
return e.get("image", "")
def _page_id(e: dict) -> tuple:
"""Identity of the PAGE an entry came from (same page ⇒ same split side)."""
img = e.get("image", "")
# pages are identified by their file; line-crop entries keep it too
stem = Path(img).name.split("__")[0]
return stem, img
def _sha256(p: Path) -> str:
h = hashlib.sha256()
with open(p, "rb") as f:
for chunk in iter(lambda: f.read(1 << 20), b""):
h.update(chunk)
return h.hexdigest()
def cmd_linepairs(args) -> None:
img_exts = {".png", ".jpg", ".jpeg", ".tif", ".tiff", ".bmp"}
entries, missing_text, empty = [], 0, 0
for p in sorted(Path(args.dir).iterdir()):
if p.suffix.lower() not in img_exts:
continue
txt_candidates = [p.with_suffix(".txt"), p.with_suffix(".TXT")]
txt = next((t for t in txt_candidates), None)
# also sibling dir variant: images/ + txt/ trees
if txt is None or not txt.exists():
alt = args.dir.parent / "txt" / p.with_suffix(".txt").name
if alt.exists():
txt = alt
if txt is None or not txt.exists():
missing_text += 1
continue
text = txt.read_text(encoding="utf-8", errors="strict").strip()
if not text:
empty += 1
continue
entries.append({"image": str(p), "box": None, "text": text})
write_jsonl(args.out, entries)
print(f"linepairs -> {args.out}: {len(entries)} lines "
f"({missing_text} images without transcript skipped, {empty} empty texts)")
def cmd_pages(args) -> None:
entries = []
for r in read_jsonl(args.input):
img = r.get("image") or r.get("file")
for ln in r.get("lines", []):
entries.append({
"image": img,
"box": ln.get("box"),
"text": ln["text"],
})
write_jsonl(args.out, entries)
print(f"pages -> {args.out}: {len(entries)} lines from {len(read_jsonl(args.input))} pages")
def cmd_split(args) -> None:
entries = read_jsonl(args.manifest)
pages: dict[tuple, list[dict]] = {}
for e in entries:
pages.setdefault(_page_id(e), []).append(e)
ids = sorted(pages)
out_tr, out_va = [], []
for pid in ids:
h = int(hashlib.sha256((":".join(map(str, pid))).encode()).hexdigest(), 16) / (1 << 128)
side = out_va if h < args.val_frac else out_tr
side.extend(pages[pid])
if not out_va and len(ids) > 1:
# hash split can leave val empty on small corpora — move one whole page
# over (page purity kept) so training always has a validation signal
pid = min(ids, key=lambda k: hashlib.sha256((":".join(map(str, k))).encode()).hexdigest())
out_tr = [e for e in out_tr if _page_id(e) != pid]
out_va = pages[pid]
print(f"note: hash split left val empty; moved page {pid[0]!r} to val")
write_jsonl(args.train_out, out_tr)
write_jsonl(args.val_out, out_va)
print(f"split -> {args.train_out} ({len(out_tr)} lines) / {args.val_out} ({len(out_va)} lines)"
f" over {len(ids)} distinct pages")
def cmd_exclude(args) -> None:
"""val pages matching (by content hash) ANY listed deployed-train page are dropped."""
train_hashes = set()
listed = 0
for line in Path(args.deployed_pages).read_text().splitlines():
if not line.strip():
continue
listed += 1
p = Path(line.strip())
if p.exists():
train_hashes.add(_sha256(p))
print(f"hashing deployed pages: {listed} listed, {len(train_hashes)} hashes read")
dropped, kept = 0, 0
out = []
seen_pages: dict[tuple, str] = {}
for e in read_jsonl(args.manifest):
pid = _page_id(e)
if pid not in seen_pages:
src = Path(e["image"])
seen_pages[pid] = _sha256(src) if src.exists() else f"missing:{src}"
if seen_pages[pid] in train_hashes:
dropped += 1
else:
out.append(e)
kept = len(out)
write_jsonl(args.out, out)
print(f"exclude -> {args.out}: dropped {dropped} lines ({kept} kept); "
f"train page hashes: {len(train_hashes)}")
def main(argv=None):
ap = argparse.ArgumentParser("data_prep")
sub = ap.add_subparsers(dest="cmd", required=True)
s = sub.add_parser("linepairs")
s.add_argument("--dir", type=Path, required=True, help="dir with line images (+.txt)")
s.add_argument("--out", type=Path, required=True)
s.set_defaults(fn=cmd_linepairs)
s = sub.add_parser("pages")
s.add_argument("--input", type=Path, required=True, help="pages jsonl with box+text")
s.add_argument("--out", type=Path, required=True)
s.set_defaults(fn=cmd_pages)
s = sub.add_parser("split")
s.add_argument("--manifest", type=Path, required=True)
s.add_argument("--train-out", type=Path, required=True)
s.add_argument("--val-out", type=Path, required=True)
s.add_argument("--val-frac", type=float, default=0.1)
s.set_defaults(fn=cmd_split)
s = sub.add_parser("exclude")
s.add_argument("--manifest", type=Path, required=True, help="val manifest to filter")
s.add_argument("--deployed-pages", type=Path, required=True,
help="text file: one path per line of deployed TRAIN page images")
s.add_argument("--out", type=Path, required=True)
s.set_defaults(fn=cmd_exclude)
args = ap.parse_args(argv)
args.fn(args)
if __name__ == "__main__":
main()