"""Build train/val manifests for Experiment B + leakage checks (plan §12). Format-agnostic entry points — the remote corpus layout is probe 5; these subcommands cover the layouts likely to exist, all emitting ONE format: {"image": ..., "box": [x0,y0,x1,y1]|null, "text": "..."} (jsonl) linepairs dir of line images + sibling transcripts (.txt files with the same stem, English names first-match) → manifest pages pages + per-line jsonl ({"image","lines":[{"box","text"}]}) → flat line manifest (boxes kept if present) split manifest → train/val jsonl by PAGE hash (0.9/0.1 default), so multi-line pages never straddle the split (§12.1 set purity) exclude remove val pages that sha256-match any page from the DEPLOYED pipeline's training renders (input: list of those image files) """ from __future__ import annotations import argparse import hashlib import json import sys from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent)) from common import read_jsonl, write_jsonl # noqa: E402 def page_key(e: dict) -> str: return e.get("image", "") def _page_id(e: dict) -> tuple: """Identity of the PAGE an entry came from (same page ⇒ same split side).""" img = e.get("image", "") # pages are identified by their file; line-crop entries keep it too stem = Path(img).name.split("__")[0] return stem, img def _sha256(p: Path) -> str: h = hashlib.sha256() with open(p, "rb") as f: for chunk in iter(lambda: f.read(1 << 20), b""): h.update(chunk) return h.hexdigest() def cmd_linepairs(args) -> None: img_exts = {".png", ".jpg", ".jpeg", ".tif", ".tiff", ".bmp"} entries, missing_text, empty = [], 0, 0 for p in sorted(Path(args.dir).iterdir()): if p.suffix.lower() not in img_exts: continue txt_candidates = [p.with_suffix(".txt"), p.with_suffix(".TXT")] txt = next((t for t in txt_candidates), None) # also sibling dir variant: images/ + txt/ trees if txt is None or not txt.exists(): alt = args.dir.parent / "txt" / p.with_suffix(".txt").name if alt.exists(): txt = alt if txt is None or not txt.exists(): missing_text += 1 continue text = txt.read_text(encoding="utf-8", errors="strict").strip() if not text: empty += 1 continue entries.append({"image": str(p), "box": None, "text": text}) write_jsonl(args.out, entries) print(f"linepairs -> {args.out}: {len(entries)} lines " f"({missing_text} images without transcript skipped, {empty} empty texts)") def cmd_pages(args) -> None: entries = [] for r in read_jsonl(args.input): img = r.get("image") or r.get("file") for ln in r.get("lines", []): entries.append({ "image": img, "box": ln.get("box"), "text": ln["text"], }) write_jsonl(args.out, entries) print(f"pages -> {args.out}: {len(entries)} lines from {len(read_jsonl(args.input))} pages") def cmd_split(args) -> None: entries = read_jsonl(args.manifest) pages: dict[tuple, list[dict]] = {} for e in entries: pages.setdefault(_page_id(e), []).append(e) ids = sorted(pages) out_tr, out_va = [], [] for pid in ids: h = int(hashlib.sha256((":".join(map(str, pid))).encode()).hexdigest(), 16) / (1 << 128) side = out_va if h < args.val_frac else out_tr side.extend(pages[pid]) if not out_va and len(ids) > 1: # hash split can leave val empty on small corpora — move one whole page # over (page purity kept) so training always has a validation signal pid = min(ids, key=lambda k: hashlib.sha256((":".join(map(str, k))).encode()).hexdigest()) out_tr = [e for e in out_tr if _page_id(e) != pid] out_va = pages[pid] print(f"note: hash split left val empty; moved page {pid[0]!r} to val") write_jsonl(args.train_out, out_tr) write_jsonl(args.val_out, out_va) print(f"split -> {args.train_out} ({len(out_tr)} lines) / {args.val_out} ({len(out_va)} lines)" f" over {len(ids)} distinct pages") def cmd_exclude(args) -> None: """val pages matching (by content hash) ANY listed deployed-train page are dropped.""" train_hashes = set() listed = 0 for line in Path(args.deployed_pages).read_text().splitlines(): if not line.strip(): continue listed += 1 p = Path(line.strip()) if p.exists(): train_hashes.add(_sha256(p)) print(f"hashing deployed pages: {listed} listed, {len(train_hashes)} hashes read") dropped, kept = 0, 0 out = [] seen_pages: dict[tuple, str] = {} for e in read_jsonl(args.manifest): pid = _page_id(e) if pid not in seen_pages: src = Path(e["image"]) seen_pages[pid] = _sha256(src) if src.exists() else f"missing:{src}" if seen_pages[pid] in train_hashes: dropped += 1 else: out.append(e) kept = len(out) write_jsonl(args.out, out) print(f"exclude -> {args.out}: dropped {dropped} lines ({kept} kept); " f"train page hashes: {len(train_hashes)}") def main(argv=None): ap = argparse.ArgumentParser("data_prep") sub = ap.add_subparsers(dest="cmd", required=True) s = sub.add_parser("linepairs") s.add_argument("--dir", type=Path, required=True, help="dir with line images (+.txt)") s.add_argument("--out", type=Path, required=True) s.set_defaults(fn=cmd_linepairs) s = sub.add_parser("pages") s.add_argument("--input", type=Path, required=True, help="pages jsonl with box+text") s.add_argument("--out", type=Path, required=True) s.set_defaults(fn=cmd_pages) s = sub.add_parser("split") s.add_argument("--manifest", type=Path, required=True) s.add_argument("--train-out", type=Path, required=True) s.add_argument("--val-out", type=Path, required=True) s.add_argument("--val-frac", type=float, default=0.1) s.set_defaults(fn=cmd_split) s = sub.add_parser("exclude") s.add_argument("--manifest", type=Path, required=True, help="val manifest to filter") s.add_argument("--deployed-pages", type=Path, required=True, help="text file: one path per line of deployed TRAIN page images") s.add_argument("--out", type=Path, required=True) s.set_defaults(fn=cmd_exclude) args = ap.parse_args(argv) args.fn(args) if __name__ == "__main__": main()