Add 1000 files (ocr-tech/tests/test_audit_limbu_book_missed_line_review_coverage.py ...)
d82db76 verified Download ocr-tech/tools/deva_rec_torch/extract_nvidia_en.py from prosperious/ocr-projects-archive: direct link, hf CLI and curl.
- Browser
- Download file 3.59 kB
-
https://huggingface.co/prosperious/ocr-projects-archive/resolve/main/ocr-tech/tools/deva_rec_torch/extract_nvidia_en.py
- Command line
-
hf download hf://prosperious/ocr-projects-archive/ocr-tech/tools/deva_rec_torch/extract_nvidia_en.py
-
curl -L -o extract_nvidia_en.py https://huggingface.co/prosperious/ocr-projects-archive/resolve/main/ocr-tech/tools/deva_rec_torch/extract_nvidia_en.py
3.59 kB
| """Extract English line crops from one NVIDIA OCR-Synthetic-Multilingual-v1 HDF5 shard. | |
| Usage: extract_nvidia_en.py <shard.h5> <dict.txt> <out_dir> [max_lines] [max_per_page] [max_len] | |
| Source licence: CC BY 4.0 (nvidia/OCR-Synthetic-Multilingual-v1); every crop's provenance | |
| (shard, page, line, bbox) is written to provenance.jsonl for attribution. Only lines with | |
| at most ``max_len`` (default 25) codepoints whose characters are all in the recognizer dictionary are kept. | |
| """ | |
| from __future__ import annotations | |
| import io | |
| import json | |
| import sys | |
| import unicodedata | |
| from pathlib import Path | |
| import h5py | |
| import numpy as np | |
| from PIL import Image | |
| def as_bytes(value) -> bytes: | |
| if isinstance(value, bytes): | |
| return value | |
| if isinstance(value, np.ndarray): | |
| return value.tobytes() | |
| raise TypeError(f"unsupported HDF5 image value {type(value)!r}") | |
| def as_json(value) -> dict: | |
| raw = value.decode("utf-8") if isinstance(value, (bytes, np.bytes_)) else str(value) | |
| return json.loads(raw) | |
| def main() -> int: | |
| shard, dict_path, out = Path(sys.argv[1]), Path(sys.argv[2]), Path(sys.argv[3]) | |
| max_lines = int(sys.argv[4]) if len(sys.argv) > 4 else 15000 | |
| max_per_page = int(sys.argv[5]) if len(sys.argv) > 5 else 5 | |
| max_len = int(sys.argv[6]) if len(sys.argv) > 6 else 25 | |
| allowed = set(line.rstrip("\n") for line in dict_path.read_text(encoding="utf-8").splitlines()) | {" "} | |
| (out / "images").mkdir(parents=True, exist_ok=True) | |
| kept = rejected_len = rejected_chars = 0 | |
| with h5py.File(shard, "r") as handle, (out / "labels.txt").open("w", encoding="utf-8") as labels, \ | |
| (out / "provenance.jsonl").open("w", encoding="utf-8") as prov: | |
| images, annotations = handle["images"], handle["annotations"] | |
| if len(images) != len(annotations): | |
| raise ValueError(f"{shard}: images/annotations length mismatch") | |
| for page in range(len(images)): | |
| if kept >= max_lines: | |
| break | |
| boxes = as_json(annotations[page]).get("line_bboxes") | |
| if not isinstance(boxes, list): | |
| raise ValueError(f"{shard} page {page}: no line_bboxes list") | |
| page_image = None | |
| taken = 0 | |
| for line_no, box in enumerate(boxes): | |
| if taken >= max_per_page or kept >= max_lines: | |
| break | |
| text = unicodedata.normalize("NFC", str(box.get("text") or "")).strip() | |
| if not text or len(text) > max_len: | |
| rejected_len += 1 | |
| continue | |
| if any(ch not in allowed for ch in text): | |
| rejected_chars += 1 | |
| continue | |
| if page_image is None: | |
| page_image = Image.open(io.BytesIO(as_bytes(images[page]))).convert("RGB") | |
| x, y, w, h = (int(round(float(v))) for v in box["bbox"]) | |
| if w < 4 or h < 4: | |
| continue | |
| crop = page_image.crop((x, y, x + w, y + h)) | |
| name = f"nv_en_{kept:06d}.png" | |
| crop.save(out / "images" / name) | |
| labels.write(f"images/{name}\t{text}\n") | |
| prov.write(json.dumps({"image": f"images/{name}", "shard": shard.name, "page": page, | |
| "line": line_no, "bbox": [x, y, w, h]}) + "\n") | |
| kept += 1 | |
| taken += 1 | |
| print(json.dumps({"kept": kept, "rejected_length": rejected_len, "rejected_chars": rejected_chars})) | |
| return 0 | |
| if __name__ == "__main__": | |
| sys.exit(main()) | |