File size: 6,704 Bytes
f9bc0c1 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 | """Build train/val manifests for Experiment B + leakage checks (plan §12).
Format-agnostic entry points — the remote corpus layout is probe 5; these
subcommands cover the layouts likely to exist, all emitting ONE format:
{"image": ..., "box": [x0,y0,x1,y1]|null, "text": "..."} (jsonl)
linepairs dir of line images + sibling transcripts (.txt files with the
same stem, English names first-match) → manifest
pages pages + per-line jsonl ({"image","lines":[{"box","text"}]}) →
flat line manifest (boxes kept if present)
split manifest → train/val jsonl by PAGE hash (0.9/0.1 default), so
multi-line pages never straddle the split (§12.1 set purity)
exclude remove val pages that sha256-match any page from the DEPLOYED
pipeline's training renders (input: list of those image files)
"""
from __future__ import annotations
import argparse
import hashlib
import json
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent))
from common import read_jsonl, write_jsonl # noqa: E402
def page_key(e: dict) -> str:
return e.get("image", "")
def _page_id(e: dict) -> tuple:
"""Identity of the PAGE an entry came from (same page ⇒ same split side)."""
img = e.get("image", "")
# pages are identified by their file; line-crop entries keep it too
stem = Path(img).name.split("__")[0]
return stem, img
def _sha256(p: Path) -> str:
h = hashlib.sha256()
with open(p, "rb") as f:
for chunk in iter(lambda: f.read(1 << 20), b""):
h.update(chunk)
return h.hexdigest()
def cmd_linepairs(args) -> None:
img_exts = {".png", ".jpg", ".jpeg", ".tif", ".tiff", ".bmp"}
entries, missing_text, empty = [], 0, 0
for p in sorted(Path(args.dir).iterdir()):
if p.suffix.lower() not in img_exts:
continue
txt_candidates = [p.with_suffix(".txt"), p.with_suffix(".TXT")]
txt = next((t for t in txt_candidates), None)
# also sibling dir variant: images/ + txt/ trees
if txt is None or not txt.exists():
alt = args.dir.parent / "txt" / p.with_suffix(".txt").name
if alt.exists():
txt = alt
if txt is None or not txt.exists():
missing_text += 1
continue
text = txt.read_text(encoding="utf-8", errors="strict").strip()
if not text:
empty += 1
continue
entries.append({"image": str(p), "box": None, "text": text})
write_jsonl(args.out, entries)
print(f"linepairs -> {args.out}: {len(entries)} lines "
f"({missing_text} images without transcript skipped, {empty} empty texts)")
def cmd_pages(args) -> None:
entries = []
for r in read_jsonl(args.input):
img = r.get("image") or r.get("file")
for ln in r.get("lines", []):
entries.append({
"image": img,
"box": ln.get("box"),
"text": ln["text"],
})
write_jsonl(args.out, entries)
print(f"pages -> {args.out}: {len(entries)} lines from {len(read_jsonl(args.input))} pages")
def cmd_split(args) -> None:
entries = read_jsonl(args.manifest)
pages: dict[tuple, list[dict]] = {}
for e in entries:
pages.setdefault(_page_id(e), []).append(e)
ids = sorted(pages)
out_tr, out_va = [], []
for pid in ids:
h = int(hashlib.sha256((":".join(map(str, pid))).encode()).hexdigest(), 16) / (1 << 128)
side = out_va if h < args.val_frac else out_tr
side.extend(pages[pid])
if not out_va and len(ids) > 1:
# hash split can leave val empty on small corpora — move one whole page
# over (page purity kept) so training always has a validation signal
pid = min(ids, key=lambda k: hashlib.sha256((":".join(map(str, k))).encode()).hexdigest())
out_tr = [e for e in out_tr if _page_id(e) != pid]
out_va = pages[pid]
print(f"note: hash split left val empty; moved page {pid[0]!r} to val")
write_jsonl(args.train_out, out_tr)
write_jsonl(args.val_out, out_va)
print(f"split -> {args.train_out} ({len(out_tr)} lines) / {args.val_out} ({len(out_va)} lines)"
f" over {len(ids)} distinct pages")
def cmd_exclude(args) -> None:
"""val pages matching (by content hash) ANY listed deployed-train page are dropped."""
train_hashes = set()
listed = 0
for line in Path(args.deployed_pages).read_text().splitlines():
if not line.strip():
continue
listed += 1
p = Path(line.strip())
if p.exists():
train_hashes.add(_sha256(p))
print(f"hashing deployed pages: {listed} listed, {len(train_hashes)} hashes read")
dropped, kept = 0, 0
out = []
seen_pages: dict[tuple, str] = {}
for e in read_jsonl(args.manifest):
pid = _page_id(e)
if pid not in seen_pages:
src = Path(e["image"])
seen_pages[pid] = _sha256(src) if src.exists() else f"missing:{src}"
if seen_pages[pid] in train_hashes:
dropped += 1
else:
out.append(e)
kept = len(out)
write_jsonl(args.out, out)
print(f"exclude -> {args.out}: dropped {dropped} lines ({kept} kept); "
f"train page hashes: {len(train_hashes)}")
def main(argv=None):
ap = argparse.ArgumentParser("data_prep")
sub = ap.add_subparsers(dest="cmd", required=True)
s = sub.add_parser("linepairs")
s.add_argument("--dir", type=Path, required=True, help="dir with line images (+.txt)")
s.add_argument("--out", type=Path, required=True)
s.set_defaults(fn=cmd_linepairs)
s = sub.add_parser("pages")
s.add_argument("--input", type=Path, required=True, help="pages jsonl with box+text")
s.add_argument("--out", type=Path, required=True)
s.set_defaults(fn=cmd_pages)
s = sub.add_parser("split")
s.add_argument("--manifest", type=Path, required=True)
s.add_argument("--train-out", type=Path, required=True)
s.add_argument("--val-out", type=Path, required=True)
s.add_argument("--val-frac", type=float, default=0.1)
s.set_defaults(fn=cmd_split)
s = sub.add_parser("exclude")
s.add_argument("--manifest", type=Path, required=True, help="val manifest to filter")
s.add_argument("--deployed-pages", type=Path, required=True,
help="text file: one path per line of deployed TRAIN page images")
s.add_argument("--out", type=Path, required=True)
s.set_defaults(fn=cmd_exclude)
args = ap.parse_args(argv)
args.fn(args)
if __name__ == "__main__":
main()
|