"""Extract English line crops from one NVIDIA OCR-Synthetic-Multilingual-v1 HDF5 shard. Usage: extract_nvidia_en.py [max_lines] [max_per_page] [max_len] Source licence: CC BY 4.0 (nvidia/OCR-Synthetic-Multilingual-v1); every crop's provenance (shard, page, line, bbox) is written to provenance.jsonl for attribution. Only lines with at most ``max_len`` (default 25) codepoints whose characters are all in the recognizer dictionary are kept. """ from __future__ import annotations import io import json import sys import unicodedata from pathlib import Path import h5py import numpy as np from PIL import Image def as_bytes(value) -> bytes: if isinstance(value, bytes): return value if isinstance(value, np.ndarray): return value.tobytes() raise TypeError(f"unsupported HDF5 image value {type(value)!r}") def as_json(value) -> dict: raw = value.decode("utf-8") if isinstance(value, (bytes, np.bytes_)) else str(value) return json.loads(raw) def main() -> int: shard, dict_path, out = Path(sys.argv[1]), Path(sys.argv[2]), Path(sys.argv[3]) max_lines = int(sys.argv[4]) if len(sys.argv) > 4 else 15000 max_per_page = int(sys.argv[5]) if len(sys.argv) > 5 else 5 max_len = int(sys.argv[6]) if len(sys.argv) > 6 else 25 allowed = set(line.rstrip("\n") for line in dict_path.read_text(encoding="utf-8").splitlines()) | {" "} (out / "images").mkdir(parents=True, exist_ok=True) kept = rejected_len = rejected_chars = 0 with h5py.File(shard, "r") as handle, (out / "labels.txt").open("w", encoding="utf-8") as labels, \ (out / "provenance.jsonl").open("w", encoding="utf-8") as prov: images, annotations = handle["images"], handle["annotations"] if len(images) != len(annotations): raise ValueError(f"{shard}: images/annotations length mismatch") for page in range(len(images)): if kept >= max_lines: break boxes = as_json(annotations[page]).get("line_bboxes") if not isinstance(boxes, list): raise ValueError(f"{shard} page {page}: no line_bboxes list") page_image = None taken = 0 for line_no, box in enumerate(boxes): if taken >= max_per_page or kept >= max_lines: break text = unicodedata.normalize("NFC", str(box.get("text") or "")).strip() if not text or len(text) > max_len: rejected_len += 1 continue if any(ch not in allowed for ch in text): rejected_chars += 1 continue if page_image is None: page_image = Image.open(io.BytesIO(as_bytes(images[page]))).convert("RGB") x, y, w, h = (int(round(float(v))) for v in box["bbox"]) if w < 4 or h < 4: continue crop = page_image.crop((x, y, x + w, y + h)) name = f"nv_en_{kept:06d}.png" crop.save(out / "images" / name) labels.write(f"images/{name}\t{text}\n") prov.write(json.dumps({"image": f"images/{name}", "shard": shard.name, "page": page, "line": line_no, "bbox": [x, y, w, h]}) + "\n") kept += 1 taken += 1 print(json.dumps({"kept": kept, "rejected_length": rejected_len, "rejected_chars": rejected_chars})) return 0 if __name__ == "__main__": sys.exit(main())