Download ocr-tech/src/ocrtech/manifest.py from prosperious/ocr-projects-archive: direct link, hf CLI and curl.
- Browser
- Download file 53.4 kB
-
https://huggingface.co/prosperious/ocr-projects-archive/resolve/main/ocr-tech/src/ocrtech/manifest.py
- Command line
-
hf download hf://prosperious/ocr-projects-archive/ocr-tech/src/ocrtech/manifest.py
-
curl -L -o manifest.py https://huggingface.co/prosperious/ocr-projects-archive/resolve/main/ocr-tech/src/ocrtech/manifest.py
53.4 kB
"""Dataset verification and manifest conversion."""
from __future__ import annotations
import csv
import hashlib
import io
import json
import shutil
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, Iterable, Iterator
from .errors import DataValidationError
from .normalization import normalize_ocr_text, unsupported_characters
IMAGE_FIELD_CANDIDATES = ("image", "image_path", "path", "file", "filename", "img")
TEXT_FIELD_CANDIDATES = ("text", "label", "transcription", "ground_truth", "gt", "clean_text")
SUPPORTED_OCR_IMAGE_SUFFIXES = frozenset({".bmp", ".dib", ".jpeg", ".jpg", ".png", ".webp", ".pbm", ".pgm", ".ppm", ".pnm", ".sr", ".ras", ".tiff", ".tif"})
@dataclass(slots=True)
class ManifestEntry:
sample_id: str
dataset: str
split: str
image_path: str
text: str
sha256: str | None = None
metadata: dict[str, Any] = field(default_factory=dict)
def __post_init__(self) -> None:
if not self.sample_id:
raise DataValidationError("manifest entry sample_id is required")
if not self.dataset:
raise DataValidationError("manifest entry dataset is required")
if not self.split:
raise DataValidationError("manifest entry split is required")
if not self.image_path:
raise DataValidationError(f"manifest entry {self.sample_id} image_path is required")
if not isinstance(self.text, str):
raise DataValidationError(f"manifest entry {self.sample_id} text must be a string")
def to_dict(self) -> dict[str, Any]:
return {
"sample_id": self.sample_id,
"dataset": self.dataset,
"split": self.split,
"image_path": self.image_path,
"text": self.text,
"sha256": self.sha256,
"metadata": self.metadata,
}
@classmethod
def from_dict(cls, data: dict[str, Any]) -> "ManifestEntry":
return cls(
sample_id=str(data.get("sample_id") or data.get("id") or ""),
dataset=str(data.get("dataset") or ""),
split=str(data.get("split") or ""),
image_path=str(data.get("image_path") or data.get("image") or ""),
text=str(data.get("text") if data.get("text") is not None else ""),
sha256=data.get("sha256"),
metadata=dict(data.get("metadata") or {}),
)
@dataclass(slots=True)
class PrepareReject:
row_index: int
reason: str
row: dict[str, Any]
def to_dict(self) -> dict[str, Any]:
return {
"row_index": self.row_index,
"reason": self.reason,
"row": self.row,
}
@dataclass(slots=True)
class PrepareReport:
manifest_path: str
sample_count: int
rejected_count: int
rejects_path: str | None
summary_json_path: str
summary_md_path: str
def to_dict(self) -> dict[str, Any]:
return {
"manifest_path": self.manifest_path,
"sample_count": self.sample_count,
"rejected_count": self.rejected_count,
"rejects_path": self.rejects_path,
"summary_json_path": self.summary_json_path,
"summary_md_path": self.summary_md_path,
}
@dataclass(slots=True)
class HfDatasetInspection:
dataset_id: str
split: str
subset: str | None
inspection_mode: str
fields: list[str]
inferred_image_field: str | None
inferred_text_field: str | None
inspected_count: int
samples: list[dict[str, Any]]
issues: list[str] = field(default_factory=list)
warnings: list[str] = field(default_factory=list)
output_json_path: str | None = None
output_md_path: str | None = None
@property
def passed(self) -> bool:
return not self.issues
def to_dict(self) -> dict[str, Any]:
return {
"dataset_id": self.dataset_id,
"split": self.split,
"subset": self.subset,
"inspection_mode": self.inspection_mode,
"passed": self.passed,
"fields": self.fields,
"inferred_image_field": self.inferred_image_field,
"inferred_text_field": self.inferred_text_field,
"inspected_count": self.inspected_count,
"samples": self.samples,
"issues": self.issues,
"warnings": self.warnings,
"output_json_path": self.output_json_path,
"output_md_path": self.output_md_path,
}
@dataclass(slots=True)
class ManifestImageNormalizationReport:
source_manifest: str
manifest_path: str
image_dir: str
sample_count: int
copied_count: int
converted_count: int
unsupported_suffixes: dict[str, int]
summary_json_path: str
summary_md_path: str
def to_dict(self) -> dict[str, Any]:
return {
"source_manifest": self.source_manifest,
"manifest_path": self.manifest_path,
"image_dir": self.image_dir,
"sample_count": self.sample_count,
"copied_count": self.copied_count,
"converted_count": self.converted_count,
"unsupported_suffixes": self.unsupported_suffixes,
"summary_json_path": self.summary_json_path,
"summary_md_path": self.summary_md_path,
}
def sha256_text(text: str) -> str:
return hashlib.sha256(text.encode("utf-8")).hexdigest()
def sha256_file(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def sha256_path(path: Path) -> str:
if path.is_file():
return sha256_file(path)
if not path.is_dir():
raise DataValidationError(f"cannot hash missing or unsupported path: {path}")
digest = hashlib.sha256()
file_paths = sorted(item for item in path.rglob("*") if item.is_file())
for file_path in file_paths:
digest.update(str(file_path.relative_to(path)).encode("utf-8"))
digest.update(b"\0")
with file_path.open("rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(chunk)
digest.update(b"\0")
return digest.hexdigest()
def read_jsonl(path: Path) -> Iterator[dict[str, Any]]:
with path.open("r", encoding="utf-8") as handle:
for line_number, line in enumerate(handle, start=1):
if not line.strip():
continue
try:
value = json.loads(line)
except json.JSONDecodeError as exc:
raise DataValidationError(f"Invalid JSONL at {path}:{line_number}: {exc}") from exc
if not isinstance(value, dict):
raise DataValidationError(f"JSONL row at {path}:{line_number} must be an object")
yield value
def read_csv_rows(path: Path) -> Iterator[dict[str, Any]]:
with path.open("r", encoding="utf-8", newline="") as handle:
reader = csv.DictReader(handle)
if not reader.fieldnames:
raise DataValidationError(f"CSV has no header: {path}")
for row in reader:
yield dict(row)
def load_manifest(path: str | Path) -> list[ManifestEntry]:
manifest_path = Path(path)
entries = [ManifestEntry.from_dict(row) for row in read_jsonl(manifest_path)]
seen: set[str] = set()
for entry in entries:
if entry.sample_id in seen:
raise DataValidationError(f"duplicate sample_id in manifest {manifest_path}: {entry.sample_id}")
seen.add(entry.sample_id)
return entries
def write_manifest(entries: Iterable[ManifestEntry], path: Path) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
with path.open("w", encoding="utf-8") as handle:
for entry in entries:
handle.write(json.dumps(entry.to_dict(), ensure_ascii=False) + "\n")
def normalize_manifest_images(
manifest_path: str | Path,
output_dir: str | Path,
*,
image_dir_name: str = "images",
output_manifest_name: str = "manifest-images-normalized.jsonl",
) -> ManifestImageNormalizationReport:
source_manifest = Path(manifest_path)
entries = load_manifest(source_manifest)
if not entries:
raise DataValidationError("image normalization requires a non-empty manifest")
out = Path(output_dir)
image_dir = out / image_dir_name
image_dir.mkdir(parents=True, exist_ok=True)
rewritten: list[ManifestEntry] = []
copied_count = 0
converted_count = 0
unsupported_suffixes: dict[str, int] = {}
targets: set[Path] = set()
for entry in entries:
source_image = _resolve_manifest_image_path(entry.image_path, source_manifest.parent)
if not source_image.exists():
raise DataValidationError(f"manifest image_path does not exist for {entry.sample_id}: {source_image}")
if not source_image.is_file():
raise DataValidationError(f"image normalization only supports file image_path entries: {entry.sample_id}: {source_image}")
source_suffix = source_image.suffix.lower()
target_suffix = source_suffix if source_suffix in SUPPORTED_OCR_IMAGE_SUFFIXES else ".png"
target = image_dir / f"{_safe_manifest_filename(entry.sample_id)}{target_suffix}"
if target in targets:
raise DataValidationError(f"duplicate normalized image target for sample_id {entry.sample_id}: {target}")
targets.add(target)
metadata = dict(entry.metadata or {})
metadata["source_image_path"] = entry.image_path
metadata["source_image_sha256"] = sha256_file(source_image)
metadata["normalized_image_suffix"] = target_suffix
if source_suffix in SUPPORTED_OCR_IMAGE_SUFFIXES:
shutil.copy2(source_image, target)
copied_count += 1
metadata["image_normalization"] = "copied"
else:
unsupported_suffix = source_suffix or "<none>"
unsupported_suffixes[unsupported_suffix] = unsupported_suffixes.get(unsupported_suffix, 0) + 1
_convert_image_to_png(source_image, target, sample_id=entry.sample_id)
converted_count += 1
metadata["image_normalization"] = "converted_to_png"
metadata["original_image_suffix"] = unsupported_suffix
rewritten.append(
ManifestEntry(
sample_id=entry.sample_id,
dataset=entry.dataset,
split=entry.split,
image_path=str(target),
text=entry.text,
sha256=sha256_file(target),
metadata=metadata,
)
)
manifest_out = out / output_manifest_name
write_manifest(rewritten, manifest_out)
summary_json = out / "image-normalization.json"
summary_md = out / "image-normalization.md"
report = ManifestImageNormalizationReport(
source_manifest=str(source_manifest),
manifest_path=str(manifest_out),
image_dir=str(image_dir),
sample_count=len(rewritten),
copied_count=copied_count,
converted_count=converted_count,
unsupported_suffixes=dict(sorted(unsupported_suffixes.items())),
summary_json_path=str(summary_json),
summary_md_path=str(summary_md),
)
summary_json.write_text(json.dumps(report.to_dict(), ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
_write_image_normalization_markdown(summary_md, report)
return report
def _resolve_manifest_image_path(path_value: str, manifest_dir: Path) -> Path:
raw = Path(path_value)
if raw.exists() or raw.is_absolute():
return raw
return manifest_dir / raw
def infer_field(row: dict[str, Any], explicit: str | None, candidates: tuple[str, ...], role: str) -> str:
if explicit:
if explicit not in row:
raise DataValidationError(f"Requested {role} field {explicit!r} is missing. Available fields: {sorted(row)}")
return explicit
present = [field for field in candidates if field in row]
matches = [field for field in present if _has_field_value(row[field])]
if len(matches) == 1:
return matches[0]
if len(matches) > 1:
return matches[0]
if len(present) == 1:
return present[0]
if len(present) > 1:
return present[0]
raise DataValidationError(
f"Could not infer {role} field. Available fields: {sorted(row)}. "
f"Pass --{role.replace('_', '-')}-field explicitly."
)
def _has_field_value(value: Any) -> bool:
if value is None:
return False
if isinstance(value, str):
return value != ""
return True
def _safe_manifest_filename(value: str) -> str:
safe = "".join(char if char.isalnum() or char in {"-", "_", "."} else "_" for char in value.strip())
return safe or "sample"
def _convert_image_to_png(source: Path, target: Path, *, sample_id: str) -> None:
try:
from PIL import Image
except ImportError as exc:
raise DataValidationError("Pillow is required to convert unsupported manifest image suffixes") from exc
try:
with Image.open(source) as image:
image.save(target, format="PNG")
except Exception as exc:
raise DataValidationError(f"failed to convert image for {sample_id} to PNG: {source}: {exc}") from exc
def _write_image_normalization_markdown(path: Path, report: ManifestImageNormalizationReport) -> None:
unsupported = ", ".join(f"`{suffix}`={count}" for suffix, count in report.unsupported_suffixes.items()) or "none"
path.write_text(
"\n".join(
[
"# Manifest Image Normalization",
"",
f"Source manifest: `{report.source_manifest}`",
f"Rewritten manifest: `{report.manifest_path}`",
f"Image directory: `{report.image_dir}`",
f"Samples: `{report.sample_count}`",
f"Copied images: `{report.copied_count}`",
f"Converted images: `{report.converted_count}`",
f"Unsupported suffixes converted: {unsupported}",
"",
]
),
encoding="utf-8",
)
def local_rows(source: Path) -> Iterator[dict[str, Any]]:
if source.is_dir():
for candidate in (source / "labels.jsonl", source / "manifest.jsonl", source / "labels.csv", source / "metadata.csv"):
if candidate.exists():
yield from local_rows(candidate)
return
raise DataValidationError(f"Directory source {source} must contain labels.jsonl, manifest.jsonl, labels.csv, or metadata.csv")
suffix = source.suffix.lower()
if suffix == ".jsonl":
yield from read_jsonl(source)
elif suffix == ".csv":
yield from read_csv_rows(source)
else:
raise DataValidationError(f"Unsupported local source format {source}. Use JSONL, CSV, or a labeled directory.")
def rows_to_manifest(
rows: Iterable[dict[str, Any]],
*,
dataset: str,
split: str,
base_dir: Path | None,
image_field: str | None = None,
text_field: str | None = None,
limit: int | None = None,
strict_chars: bool = False,
) -> list[ManifestEntry]:
entries, rejects = convert_rows_to_manifest(
rows,
dataset=dataset,
split=split,
base_dir=base_dir,
image_field=image_field,
text_field=text_field,
limit=limit,
strict_chars=strict_chars,
skip_invalid=False,
)
if rejects:
raise DataValidationError(f"Unexpected rejected rows while preparing {dataset} {split}")
if not entries:
raise DataValidationError("No rows converted into manifest entries")
return entries
def convert_rows_to_manifest(
rows: Iterable[dict[str, Any]],
*,
dataset: str,
split: str,
base_dir: Path | None,
image_field: str | None = None,
text_field: str | None = None,
limit: int | None = None,
strict_chars: bool = False,
skip_invalid: bool = False,
) -> tuple[list[ManifestEntry], list[PrepareReject]]:
entries: list[ManifestEntry] = []
rejects: list[PrepareReject] = []
for index, row in enumerate(rows):
if limit is not None and index >= limit:
break
if not isinstance(row, dict):
error = DataValidationError(f"Row {index} must be an object")
if not skip_invalid:
raise error
rejects.append(PrepareReject(row_index=index, reason=str(error), row={"_raw": repr(row)}))
continue
try:
entries.append(
_manifest_entry_from_row(
row,
row_index=index,
dataset=dataset,
split=split,
base_dir=base_dir,
image_field=image_field,
text_field=text_field,
strict_chars=strict_chars,
)
)
except DataValidationError as exc:
if not skip_invalid:
raise
rejects.append(PrepareReject(row_index=index, reason=str(exc), row=dict(row)))
return entries, rejects
def _manifest_entry_from_row(
row: dict[str, Any],
*,
row_index: int,
dataset: str,
split: str,
base_dir: Path | None,
image_field: str | None,
text_field: str | None,
strict_chars: bool,
) -> ManifestEntry:
image_key = infer_field(row, image_field, IMAGE_FIELD_CANDIDATES, "image")
text_key = infer_field(row, text_field, TEXT_FIELD_CANDIDATES, "text")
image_value = row[image_key]
if not _has_field_value(image_value):
raise DataValidationError(f"Row {row_index} image field {image_key!r} is empty")
raw_text = row[text_key]
if not _has_field_value(raw_text):
raise DataValidationError(f"Row {row_index} text field {text_key!r} is empty")
text = normalize_ocr_text(str(raw_text))
if not text:
raise DataValidationError(f"Row {row_index} normalized text is empty")
if strict_chars:
unsupported = unsupported_characters(text)
if unsupported:
raise DataValidationError(f"Row {row_index} contains unsupported characters: {unsupported}")
image_path = Path(str(image_value))
if base_dir is not None and not image_path.is_absolute():
image_path = base_dir / image_path
if not image_path.exists():
raise DataValidationError(f"Row {row_index} image does not exist: {image_path}")
explicit_sample_id = row.get("sample_id")
if _has_field_value(explicit_sample_id):
sample_id = str(explicit_sample_id)
elif _has_field_value(row.get("id")):
sample_id = f"{dataset}-{split}-{row['id']}"
else:
sample_id = f"{dataset}-{split}-{row_index:08d}"
image_sha256 = sha256_file(image_path)
text_sha256 = sha256_text(text)
return ManifestEntry(
sample_id=sample_id,
dataset=dataset,
split=split,
image_path=str(image_path),
text=text,
sha256=image_sha256,
metadata={
"source_row": row_index,
"image_field": image_key,
"text_field": text_key,
"text_sha256": text_sha256,
"sample_sha256": sha256_text(f"{image_sha256}\n{text_sha256}"),
},
)
def prepare_local_dataset(
source: str | Path,
output_dir: str | Path,
*,
dataset: str | None = None,
split: str = "train",
image_field: str | None = None,
text_field: str | None = None,
limit: int | None = None,
strict_chars: bool = False,
skip_invalid: bool = False,
slices: list[str] | None = None,
) -> PrepareReport:
source_path = Path(source)
if not source_path.exists():
raise DataValidationError(f"Local source does not exist: {source_path}")
dataset_name = dataset or source_path.stem or source_path.name
rows = local_rows(source_path)
base_dir = source_path if source_path.is_dir() else source_path.parent
entries, rejects = convert_rows_to_manifest(
rows,
dataset=dataset_name,
split=split,
base_dir=base_dir,
image_field=image_field,
text_field=text_field,
limit=limit,
strict_chars=strict_chars,
skip_invalid=skip_invalid,
)
entries = _stamp_slices(entries, slices or [])
return _finalize_prepare_report(output_dir, dataset_name, split, entries, rejects)
def prepare_hf_dataset(
dataset_id: str,
output_dir: str | Path,
*,
dataset: str | None = None,
split: str = "train",
image_field: str | None = None,
text_field: str | None = None,
hf_subset: str | None = None,
limit: int | None = None,
strict_chars: bool = False,
skip_invalid: bool = False,
slices: list[str] | None = None,
) -> PrepareReport:
if dataset_id == "prashant0919/nepali-synthetic-ocr-lines":
return _prepare_prashant_synthetic_dataset(
dataset_id,
output_dir,
dataset=dataset,
split=split,
limit=limit,
strict_chars=strict_chars,
skip_invalid=skip_invalid,
slices=slices,
)
if dataset_id == "nvidia/OCR-Synthetic-Multilingual-v1":
return _prepare_nvidia_hdf5_dataset(
dataset_id,
output_dir,
dataset=dataset,
split=split,
subset=hf_subset or "en",
limit=limit,
strict_chars=strict_chars,
slices=slices,
)
try:
from datasets import load_dataset
except ImportError as exc:
raise DataValidationError("Hugging Face dataset support requires installing ocr-tech[datasets].") from exc
try:
hf_dataset = load_dataset(dataset_id, split=split, streaming=False)
except Exception as exc:
raise DataValidationError(f"Failed to load Hugging Face dataset {dataset_id!r} split {split!r}: {exc}") from exc
output_root = Path(output_dir)
image_dir = output_root / "images" / (dataset or dataset_id.replace("/", "__")) / split
image_dir.mkdir(parents=True, exist_ok=True)
rows: list[dict[str, Any]] = []
for index, row in enumerate(hf_dataset):
if limit is not None and index >= limit:
break
if not isinstance(row, dict):
raise DataValidationError(f"HF row {index} is not an object")
image_key = infer_field(row, image_field, IMAGE_FIELD_CANDIDATES, "image")
text_key = infer_field(row, text_field, TEXT_FIELD_CANDIDATES, "text")
image_value = row[image_key]
image_path = image_dir / f"{index:08d}.png"
if isinstance(image_value, str):
candidate = Path(image_value)
if not candidate.exists():
raise DataValidationError(f"HF row {index} image path does not exist locally: {candidate}")
image_path = candidate
elif hasattr(image_value, "save"):
image_value.save(image_path)
elif isinstance(image_value, dict) and image_value.get("path"):
candidate = Path(str(image_value["path"]))
if not candidate.exists():
raise DataValidationError(f"HF row {index} image path does not exist locally: {candidate}")
image_path = candidate
else:
raise DataValidationError(f"HF row {index} has unsupported image field type: {type(image_value).__name__}")
rows.append({"image_path": str(image_path), "text": row[text_key], "id": row.get("id", index)})
dataset_name = dataset or dataset_id.replace("/", "__")
entries, rejects = convert_rows_to_manifest(
rows,
dataset=dataset_name,
split=split,
base_dir=None,
image_field="image_path",
text_field="text",
limit=None,
strict_chars=strict_chars,
skip_invalid=skip_invalid,
)
entries = _stamp_slices(entries, _default_hf_slices(dataset_id, subset=hf_subset) + (slices or []))
return _finalize_prepare_report(output_dir, dataset_name, split, entries, rejects)
def inspect_hf_dataset(
dataset_id: str,
output_dir: str | Path,
*,
split: str = "train",
hf_subset: str | None = None,
image_field: str | None = None,
text_field: str | None = None,
limit: int = 3,
) -> HfDatasetInspection:
if limit <= 0:
raise DataValidationError("inspect-hf-dataset limit must be positive")
if dataset_id == "prashant0919/nepali-synthetic-ocr-lines":
report = _inspect_prashant_synthetic_dataset(dataset_id, split=split, limit=limit)
elif dataset_id == "nvidia/OCR-Synthetic-Multilingual-v1":
report = _inspect_nvidia_hdf5_dataset(dataset_id, split=split, subset=hf_subset or "en", limit=limit)
else:
report = _inspect_generic_hf_dataset(
dataset_id,
split=split,
hf_subset=hf_subset,
image_field=image_field,
text_field=text_field,
limit=limit,
)
return _write_hf_inspection(report, Path(output_dir))
def prepare_hf_correction_pairs(
dataset_id: str,
output_path: str | Path,
*,
split: str = "train",
limit: int | None = None,
) -> Path:
if dataset_id != "cfilt/RoundTripOCR-nepali":
raise DataValidationError(f"unsupported correction dataset: {dataset_id}")
try:
from datasets import load_dataset
except ImportError as exc:
raise DataValidationError("Hugging Face correction dataset support requires installing ocr-tech[datasets].") from exc
try:
hf_dataset = load_dataset(dataset_id, split=split, streaming=False)
except Exception as exc:
raise DataValidationError(f"Failed to load Hugging Face dataset {dataset_id!r} split {split!r}: {exc}") from exc
out = Path(output_path)
out.parent.mkdir(parents=True, exist_ok=True)
count = 0
with out.open("w", encoding="utf-8") as handle:
for index, row in enumerate(hf_dataset):
if limit is not None and index >= limit:
break
if not isinstance(row, dict):
raise DataValidationError(f"HF correction row {index} is not an object")
noisy = row.get("ocr")
clean = row.get("correct")
if not isinstance(noisy, str) or not isinstance(clean, str):
raise DataValidationError(f"HF correction row {index} requires string ocr and correct fields")
handle.write(
json.dumps(
{
"noisy_text": normalize_ocr_text(noisy),
"clean_text": normalize_ocr_text(clean),
"font": row.get("font"),
"dataset": dataset_id,
"split": split,
},
ensure_ascii=False,
)
+ "\n"
)
count += 1
if count == 0:
raise DataValidationError(f"No correction pairs written for {dataset_id} split {split}")
return out
def _inspect_generic_hf_dataset(
dataset_id: str,
*,
split: str,
hf_subset: str | None,
image_field: str | None,
text_field: str | None,
limit: int,
) -> HfDatasetInspection:
try:
from datasets import load_dataset
except ImportError as exc:
raise DataValidationError("Hugging Face dataset support requires installing ocr-tech[datasets].") from exc
try:
if hf_subset:
hf_dataset = load_dataset(dataset_id, hf_subset, split=split, streaming=False)
else:
hf_dataset = load_dataset(dataset_id, split=split, streaming=False)
except Exception as exc:
raise DataValidationError(f"Failed to load Hugging Face dataset {dataset_id!r} split {split!r}: {exc}") from exc
rows: list[dict[str, Any]] = []
for index, row in enumerate(hf_dataset):
if index >= limit:
break
if not isinstance(row, dict):
raise DataValidationError(f"HF row {index} is not an object")
rows.append(row)
if not rows:
raise DataValidationError(f"No rows available for Hugging Face dataset {dataset_id!r} split {split!r}")
fields = sorted(rows[0])
issues: list[str] = []
warnings: list[str] = []
inferred_image = _try_infer_field(rows[0], image_field, IMAGE_FIELD_CANDIDATES, "image", issues)
inferred_text = _try_infer_field(rows[0], text_field, TEXT_FIELD_CANDIDATES, "text", issues)
samples = [
_summarize_hf_row(row, row_index=index, image_field=inferred_image, text_field=inferred_text)
for index, row in enumerate(rows)
]
for sample in samples:
if inferred_image and not sample.get("image_supported_by_prepare", False):
warnings.append(f"row {sample['row_index']} image field {inferred_image!r} may need a dataset-specific converter")
if inferred_text and sample.get("text_length", 0) == 0:
issues.append(f"row {sample['row_index']} text field {inferred_text!r} is empty after string conversion")
return HfDatasetInspection(
dataset_id=dataset_id,
split=split,
subset=hf_subset,
inspection_mode="datasets.load_dataset",
fields=fields,
inferred_image_field=inferred_image,
inferred_text_field=inferred_text,
inspected_count=len(samples),
samples=samples,
issues=issues,
warnings=warnings,
)
def _inspect_prashant_synthetic_dataset(dataset_id: str, *, split: str, limit: int) -> HfDatasetInspection:
prefix = f"{split}/"
files = set(_list_dataset_files(dataset_id, prefix=prefix, suffix=""))
metadata_file = f"{prefix}metadata.jsonl"
issues: list[str] = []
if metadata_file not in files:
issues.append(f"metadata.jsonl is missing: {metadata_file}")
return HfDatasetInspection(dataset_id, split, None, "huggingface_hub files", [], None, None, 0, [], issues)
metadata_path = _download_dataset_file(dataset_id, metadata_file)
samples: list[dict[str, Any]] = []
fields: set[str] = set()
for index, row in enumerate(read_jsonl(metadata_path)):
if index >= limit:
break
fields.update(row)
file_name = row.get("file_name")
dataset_file = file_name if isinstance(file_name, str) and file_name.startswith(prefix) else f"{prefix}{file_name}"
samples.append(
{
"row_index": index,
"fields": sorted(row),
"image_type": "repo_file",
"image_path": dataset_file,
"image_exists_in_repo": dataset_file in files,
"image_supported_by_prepare": dataset_file in files,
"text_type": type(row.get("text")).__name__,
"text_length": len(str(row.get("text") or "")),
"text_preview": _preview_text(row.get("text")),
}
)
if dataset_file not in files:
issues.append(f"row {index} image file missing from dataset repo: {dataset_file}")
return HfDatasetInspection(
dataset_id=dataset_id,
split=split,
subset=None,
inspection_mode="huggingface_hub metadata.jsonl",
fields=sorted(fields),
inferred_image_field="file_name",
inferred_text_field="text",
inspected_count=len(samples),
samples=samples,
issues=issues,
)
def _inspect_nvidia_hdf5_dataset(dataset_id: str, *, split: str, subset: str, limit: int) -> HfDatasetInspection:
try:
import h5py
except ImportError as exc:
raise DataValidationError("NVIDIA HDF5 dataset inspection requires installing ocr-tech[datasets].") from exc
shard_files = _list_dataset_files(dataset_id, prefix=f"{subset}/{split}/", suffix=".h5")
if not shard_files:
return HfDatasetInspection(
dataset_id,
split,
subset,
"hdf5 line_bboxes",
[],
None,
None,
0,
[],
[f"No HDF5 shards found for subset={subset!r} split={split!r}"],
)
samples: list[dict[str, Any]] = []
shard_path = _download_dataset_file(dataset_id, shard_files[0])
with h5py.File(shard_path, "r") as handle:
fields = sorted(handle.keys())
for required_key in ("images", "annotations"):
if required_key not in handle:
return HfDatasetInspection(
dataset_id,
split,
subset,
"hdf5 line_bboxes",
fields,
None,
None,
0,
[],
[f"HDF5 shard {shard_path} is missing dataset {required_key!r}"],
)
image_dataset = handle["images"]
annotation_dataset = handle["annotations"]
page_count = min(len(image_dataset), len(annotation_dataset))
for page_index in range(page_count):
if len(samples) >= limit:
break
annotation = _coerce_annotation_json(annotation_dataset[page_index], shard_path, page_index)
line_boxes = annotation.get("line_bboxes")
if not isinstance(line_boxes, list):
samples.append(
{
"row_index": page_index,
"shard_file": shard_files[0],
"annotation_keys": sorted(annotation),
"line_bbox_count": None,
"image_supported_by_prepare": False,
"text_length": 0,
"text_preview": "",
}
)
continue
for line_index, line_box in enumerate(line_boxes):
if len(samples) >= limit:
break
text = line_box.get("text") if isinstance(line_box, dict) else None
samples.append(
{
"row_index": len(samples),
"shard_file": shard_files[0],
"page_index": page_index,
"line_index": line_index,
"annotation_keys": sorted(annotation),
"line_bbox_count": len(line_boxes),
"bbox": line_box.get("bbox") if isinstance(line_box, dict) else None,
"image_type": "hdf5_page_crop",
"image_supported_by_prepare": isinstance(line_box, dict) and isinstance(line_box.get("bbox"), list),
"text_type": type(text).__name__,
"text_length": len(str(text or "")),
"text_preview": _preview_text(text),
}
)
return HfDatasetInspection(
dataset_id=dataset_id,
split=split,
subset=subset,
inspection_mode="hdf5 line_bboxes",
fields=["images", "annotations"],
inferred_image_field="images + annotations.line_bboxes[].bbox",
inferred_text_field="annotations.line_bboxes[].text",
inspected_count=len(samples),
samples=samples,
issues=[] if samples else [f"No line boxes found in first shard: {shard_files[0]}"],
warnings=[f"inspected first shard only: {shard_files[0]}"],
)
def _try_infer_field(
row: dict[str, Any],
explicit: str | None,
candidates: tuple[str, ...],
role: str,
issues: list[str],
) -> str | None:
try:
return infer_field(row, explicit, candidates, role)
except DataValidationError as exc:
issues.append(str(exc))
return None
def _summarize_hf_row(
row: dict[str, Any],
*,
row_index: int,
image_field: str | None,
text_field: str | None,
) -> dict[str, Any]:
image_value = row.get(image_field) if image_field else None
text_value = row.get(text_field) if text_field else None
return {
"row_index": row_index,
"fields": sorted(row),
"image_field": image_field,
"image_type": type(image_value).__name__,
"image_supported_by_prepare": _is_supported_hf_image_value(image_value),
"image_summary": _summarize_value(image_value),
"text_field": text_field,
"text_type": type(text_value).__name__,
"text_length": len(str(text_value or "")),
"text_preview": _preview_text(text_value),
}
def _is_supported_hf_image_value(value: Any) -> bool:
if isinstance(value, str):
return True
if hasattr(value, "save"):
return True
return isinstance(value, dict) and bool(value.get("path"))
def _summarize_value(value: Any) -> Any:
if isinstance(value, (str, int, float, bool)) or value is None:
text = str(value)
return text[:160] + ("..." if len(text) > 160 else "")
if isinstance(value, dict):
return {str(key): _summarize_value(item) for key, item in list(value.items())[:10]}
if hasattr(value, "size"):
return {"type": type(value).__name__, "size": getattr(value, "size", None), "mode": getattr(value, "mode", None)}
return {"type": type(value).__name__}
def _preview_text(value: Any, limit: int = 120) -> str:
text = normalize_ocr_text(str(value or ""))
return text[:limit] + ("..." if len(text) > limit else "")
def _write_hf_inspection(report: HfDatasetInspection, output_dir: Path) -> HfDatasetInspection:
output_dir.mkdir(parents=True, exist_ok=True)
stem = report.dataset_id.replace("/", "__")
if report.subset:
stem = f"{stem}__{report.subset}"
stem = f"{stem}-{report.split}-inspect"
json_path = output_dir / f"{stem}.json"
md_path = output_dir / f"{stem}.md"
report.output_json_path = str(json_path)
report.output_md_path = str(md_path)
json_path.write_text(json.dumps(report.to_dict(), ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
lines = [
"# Hugging Face Dataset Inspection",
"",
f"Dataset: `{report.dataset_id}`",
f"Split: `{report.split}`",
f"Subset: `{report.subset or ''}`",
f"Mode: `{report.inspection_mode}`",
f"Passed: `{report.passed}`",
f"Fields: `{', '.join(report.fields)}`",
f"Inferred image field: `{report.inferred_image_field or ''}`",
f"Inferred text field: `{report.inferred_text_field or ''}`",
f"Inspected samples: `{report.inspected_count}`",
"",
"## Issues",
"",
]
lines.extend([f"- {issue}" for issue in report.issues] or ["- none"])
lines.extend(["", "## Warnings", ""])
lines.extend([f"- {warning}" for warning in report.warnings] or ["- none"])
lines.extend(["", "## Samples", ""])
for sample in report.samples:
lines.append(f"- row `{sample.get('row_index')}`: text_len=`{sample.get('text_length')}` preview=`{sample.get('text_preview')}`")
md_path.write_text("\n".join(lines) + "\n", encoding="utf-8")
return report
def _prepare_prashant_synthetic_dataset(
dataset_id: str,
output_dir: str | Path,
*,
dataset: str | None,
split: str,
limit: int | None,
strict_chars: bool,
skip_invalid: bool,
slices: list[str] | None,
) -> PrepareReport:
prefix = f"{split}/"
files = set(_list_dataset_files(dataset_id, prefix=prefix, suffix=""))
metadata_file = f"{prefix}metadata.jsonl"
if metadata_file not in files:
raise DataValidationError(f"metadata.jsonl is missing for {dataset_id} split {split}: {metadata_file}")
metadata_path = _download_dataset_file(dataset_id, metadata_file)
rows: list[dict[str, Any]] = []
for index, row in enumerate(read_jsonl(metadata_path)):
if limit is not None and index >= limit:
break
file_name = row.get("file_name")
if not isinstance(file_name, str) or not file_name:
raise DataValidationError(f"HF row {index} in {metadata_file} requires a non-empty file_name")
dataset_file = file_name if file_name.startswith(prefix) else f"{prefix}{file_name}"
if dataset_file not in files:
raise DataValidationError(f"HF row {index} image file is missing from dataset repo: {dataset_file}")
image_path = _download_dataset_file(dataset_id, dataset_file)
rows.append(
{
"image_path": str(image_path),
"text": row.get("text"),
"id": row.get("id", index),
}
)
dataset_name = dataset or dataset_id.replace("/", "__")
entries, rejects = convert_rows_to_manifest(
rows,
dataset=dataset_name,
split=split,
base_dir=None,
image_field="image_path",
text_field="text",
limit=limit,
strict_chars=strict_chars,
skip_invalid=skip_invalid,
)
entries = _stamp_slices(entries, _default_hf_slices(dataset_id, subset=None) + (slices or []))
return _finalize_prepare_report(output_dir, dataset_name, split, entries, rejects)
def _prepare_nvidia_hdf5_dataset(
dataset_id: str,
output_dir: str | Path,
*,
dataset: str | None,
split: str,
subset: str,
limit: int | None,
strict_chars: bool,
slices: list[str] | None,
) -> PrepareReport:
try:
import h5py
except ImportError as exc:
raise DataValidationError("NVIDIA HDF5 dataset support requires installing ocr-tech[datasets].") from exc
try:
from PIL import Image
except ImportError as exc:
raise DataValidationError("NVIDIA HDF5 dataset support requires Pillow. Install ocr-tech[datasets].") from exc
shard_files = _list_dataset_files(dataset_id, prefix=f"{subset}/{split}/", suffix=".h5")
if not shard_files:
raise DataValidationError(f"No HDF5 shards found for {dataset_id} subset {subset!r} split {split!r}")
dataset_name = dataset or f"{dataset_id.replace('/', '__')}__{subset}"
output_root = Path(output_dir)
image_dir = output_root / "images" / dataset_name / split
image_dir.mkdir(parents=True, exist_ok=True)
entries: list[ManifestEntry] = []
for shard_index, shard_file in enumerate(shard_files):
shard_path = _download_dataset_file(dataset_id, shard_file)
with h5py.File(shard_path, "r") as handle:
for required_key in ("images", "annotations"):
if required_key not in handle:
raise DataValidationError(f"HDF5 shard {shard_path} is missing dataset {required_key!r}")
image_dataset = handle["images"]
annotation_dataset = handle["annotations"]
if len(image_dataset) != len(annotation_dataset):
raise DataValidationError(f"HDF5 shard {shard_path} has mismatched images and annotations lengths")
for page_index in range(len(image_dataset)):
if limit is not None and len(entries) >= limit:
break
image_bytes = _coerce_hdf5_bytes(image_dataset[page_index])
annotation = _coerce_annotation_json(annotation_dataset[page_index], shard_path, page_index)
line_boxes = annotation.get("line_bboxes")
if not isinstance(line_boxes, list):
raise DataValidationError(f"HDF5 shard {shard_path} page {page_index} annotation is missing line_bboxes")
page_image = Image.open(io.BytesIO(image_bytes)).convert("RGB")
for line_index, line_box in enumerate(line_boxes):
if limit is not None and len(entries) >= limit:
break
if not isinstance(line_box, dict):
raise DataValidationError(f"HDF5 shard {shard_path} page {page_index} line {line_index} is not an object")
text = normalize_ocr_text(str(line_box.get("text") or ""))
if not text:
continue
if strict_chars:
unsupported = unsupported_characters(text)
if unsupported:
raise DataValidationError(
f"HDF5 shard {shard_path} page {page_index} line {line_index} contains unsupported characters: {unsupported}"
)
bbox = _coerce_bbox(line_box.get("bbox"), shard_path, page_index, line_index)
crop = page_image.crop((bbox[0], bbox[1], bbox[0] + bbox[2], bbox[1] + bbox[3]))
if crop.width <= 0 or crop.height <= 0:
raise DataValidationError(f"HDF5 shard {shard_path} page {page_index} line {line_index} produced an empty crop")
file_name = f"{shard_path.stem}-p{page_index:06d}-l{line_index:04d}.png"
crop_path = image_dir / file_name
crop.save(crop_path)
image_sha256 = sha256_file(crop_path)
text_sha256 = sha256_text(text)
sample_id = f"{dataset_name}-{split}-{shard_index:03d}-{page_index:06d}-{line_index:04d}"
entries.append(
ManifestEntry(
sample_id=sample_id,
dataset=dataset_name,
split=split,
image_path=str(crop_path),
text=text,
sha256=image_sha256,
metadata={
"source_dataset": dataset_id,
"subset": subset,
"shard_path": str(shard_path),
"page_index": page_index,
"line_index": line_index,
"bbox": bbox,
"text_sha256": text_sha256,
"sample_sha256": sha256_text(f"{image_sha256}\n{text_sha256}"),
"slices": ["english", subset, "synthetic", "line_crop"],
},
)
)
if limit is not None and len(entries) >= limit:
break
if not entries:
raise DataValidationError(f"No line crops were prepared for {dataset_id} subset {subset!r} split {split!r}")
entries = _stamp_slices(entries, slices or [])
return _finalize_prepare_report(output_root, dataset_name, split, entries, [])
def _stamp_slices(entries: list[ManifestEntry], slices: list[str]) -> list[ManifestEntry]:
normalized = [str(item).strip() for item in slices if str(item).strip()]
if not normalized:
return entries
for entry in entries:
metadata = dict(entry.metadata or {})
existing = metadata.get("slices")
combined: list[str] = []
if isinstance(existing, list):
combined.extend(str(item).strip() for item in existing if str(item).strip())
elif isinstance(existing, str) and existing.strip():
combined.append(existing.strip())
combined.extend(normalized)
metadata["slices"] = sorted(set(combined))
entry.metadata = metadata
return entries
def _default_hf_slices(dataset_id: str, *, subset: str | None) -> list[str]:
if dataset_id == "gauravgiri/nepali-ocr-dataset":
return ["nepali", "real"]
if dataset_id == "prashant0919/nepali-synthetic-ocr-lines":
return ["nepali", "synthetic"]
if dataset_id == "nvidia/OCR-Synthetic-Multilingual-v1":
values = ["english", "synthetic", "line_crop"]
if subset:
values.append(subset)
return values
return []
def _finalize_prepare_report(
output_dir: str | Path,
dataset_name: str,
split: str,
entries: list[ManifestEntry],
rejects: list[PrepareReject],
) -> PrepareReport:
if not entries:
if rejects:
raise DataValidationError(f"No rows converted into manifest entries; rejected rows={len(rejects)}")
raise DataValidationError("No rows converted into manifest entries")
output_root = Path(output_dir)
output_root.mkdir(parents=True, exist_ok=True)
stem = f"{dataset_name}-{split}"
manifest_path = output_root / f"{stem}.jsonl"
rejects_path = output_root / f"{stem}-rejects.jsonl"
summary_json_path = output_root / f"{stem}-prepare.json"
summary_md_path = output_root / f"{stem}-prepare.md"
write_manifest(entries, manifest_path)
if rejects:
with rejects_path.open("w", encoding="utf-8") as handle:
for reject in rejects:
handle.write(json.dumps(reject.to_dict(), ensure_ascii=False) + "\n")
elif rejects_path.exists():
rejects_path.unlink()
report = PrepareReport(
manifest_path=str(manifest_path),
sample_count=len(entries),
rejected_count=len(rejects),
rejects_path=str(rejects_path) if rejects else None,
summary_json_path=str(summary_json_path),
summary_md_path=str(summary_md_path),
)
_write_prepare_report(report, summary_json_path, summary_md_path)
return report
def _write_prepare_report(report: PrepareReport, summary_json_path: Path, summary_md_path: Path) -> None:
summary_json_path.write_text(json.dumps(report.to_dict(), ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
lines = [
"# Dataset Prepare",
"",
f"Manifest: `{report.manifest_path}`",
f"Prepared samples: `{report.sample_count}`",
f"Rejected rows: `{report.rejected_count}`",
f"Reject log: `{report.rejects_path or ''}`",
]
summary_md_path.write_text("\n".join(lines) + "\n", encoding="utf-8")
def _snapshot_dataset(dataset_id: str, *, allow_patterns: list[str]) -> Path:
try:
from huggingface_hub import snapshot_download
except ImportError as exc:
raise DataValidationError("Snapshot download support requires installing ocr-tech[datasets].") from exc
try:
local_dir = snapshot_download(repo_id=dataset_id, repo_type="dataset", allow_patterns=allow_patterns)
except Exception as exc:
raise DataValidationError(f"Failed to snapshot dataset {dataset_id!r}: {exc}") from exc
return Path(local_dir)
def _list_dataset_files(dataset_id: str, *, prefix: str, suffix: str) -> list[str]:
try:
from huggingface_hub import list_repo_files
except ImportError as exc:
raise DataValidationError("Dataset file listing requires installing ocr-tech[datasets].") from exc
try:
files = list_repo_files(repo_id=dataset_id, repo_type="dataset")
except Exception as exc:
raise DataValidationError(f"Failed to list dataset files for {dataset_id!r}: {exc}") from exc
return sorted(path for path in files if path.startswith(prefix) and path.endswith(suffix))
def _download_dataset_file(dataset_id: str, dataset_file: str) -> Path:
try:
from huggingface_hub import hf_hub_download
except ImportError as exc:
raise DataValidationError("Dataset file download requires installing ocr-tech[datasets].") from exc
try:
local_path = hf_hub_download(repo_id=dataset_id, repo_type="dataset", filename=dataset_file)
except Exception as exc:
raise DataValidationError(f"Failed to download dataset file {dataset_file!r} from {dataset_id!r}: {exc}") from exc
return Path(local_path)
def _coerce_hdf5_bytes(value: Any) -> bytes:
if isinstance(value, bytes):
return value
if hasattr(value, "tobytes"):
return value.tobytes()
raise DataValidationError(f"Unsupported HDF5 image payload type: {type(value).__name__}")
def _coerce_annotation_json(value: Any, shard_path: Path, page_index: int) -> dict[str, Any]:
if isinstance(value, bytes):
raw = value.decode("utf-8")
elif isinstance(value, str):
raw = value
elif hasattr(value, "tobytes"):
raw = value.tobytes().decode("utf-8")
else:
raise DataValidationError(f"HDF5 shard {shard_path} page {page_index} annotation has unsupported type: {type(value).__name__}")
try:
payload = json.loads(raw)
except json.JSONDecodeError as exc:
raise DataValidationError(f"HDF5 shard {shard_path} page {page_index} annotation is not valid JSON: {exc}") from exc
if not isinstance(payload, dict):
raise DataValidationError(f"HDF5 shard {shard_path} page {page_index} annotation must be a JSON object")
return payload
def _coerce_bbox(value: Any, shard_path: Path, page_index: int, line_index: int) -> list[int]:
if not isinstance(value, list) or len(value) != 4:
raise DataValidationError(f"HDF5 shard {shard_path} page {page_index} line {line_index} bbox must be [x, y, w, h]")
bbox = [int(item) for item in value]
if bbox[2] <= 0 or bbox[3] <= 0:
raise DataValidationError(f"HDF5 shard {shard_path} page {page_index} line {line_index} bbox width/height must be positive")
return bbox
"""Dataset verification and manifest conversion."""
from __future__ import annotations
import csv
import hashlib
import io
import json
import shutil
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, Iterable, Iterator
from .errors import DataValidationError
from .normalization import normalize_ocr_text, unsupported_characters
IMAGE_FIELD_CANDIDATES = ("image", "image_path", "path", "file", "filename", "img")
TEXT_FIELD_CANDIDATES = ("text", "label", "transcription", "ground_truth", "gt", "clean_text")
SUPPORTED_OCR_IMAGE_SUFFIXES = frozenset({".bmp", ".dib", ".jpeg", ".jpg", ".png", ".webp", ".pbm", ".pgm", ".ppm", ".pnm", ".sr", ".ras", ".tiff", ".tif"})
class ManifestEntry:
sample_id: str
dataset: str
split: str
image_path: str
text: str
sha256: str | None = None
metadata: dict[str, Any] = field(default_factory=dict)
def __post_init__(self) -> None:
if not self.sample_id:
raise DataValidationError("manifest entry sample_id is required")
if not self.dataset:
raise DataValidationError("manifest entry dataset is required")
if not self.split:
raise DataValidationError("manifest entry split is required")
if not self.image_path:
raise DataValidationError(f"manifest entry {self.sample_id} image_path is required")
if not isinstance(self.text, str):
raise DataValidationError(f"manifest entry {self.sample_id} text must be a string")
def to_dict(self) -> dict[str, Any]:
return {
"sample_id": self.sample_id,
"dataset": self.dataset,
"split": self.split,
"image_path": self.image_path,
"text": self.text,
"sha256": self.sha256,
"metadata": self.metadata,
}
def from_dict(cls, data: dict[str, Any]) -> "ManifestEntry":
return cls(
sample_id=str(data.get("sample_id") or data.get("id") or ""),
dataset=str(data.get("dataset") or ""),
split=str(data.get("split") or ""),
image_path=str(data.get("image_path") or data.get("image") or ""),
text=str(data.get("text") if data.get("text") is not None else ""),
sha256=data.get("sha256"),
metadata=dict(data.get("metadata") or {}),
)
class PrepareReject:
row_index: int
reason: str
row: dict[str, Any]
def to_dict(self) -> dict[str, Any]:
return {
"row_index": self.row_index,
"reason": self.reason,
"row": self.row,
}
class PrepareReport:
manifest_path: str
sample_count: int
rejected_count: int
rejects_path: str | None
summary_json_path: str
summary_md_path: str
def to_dict(self) -> dict[str, Any]:
return {
"manifest_path": self.manifest_path,
"sample_count": self.sample_count,
"rejected_count": self.rejected_count,
"rejects_path": self.rejects_path,
"summary_json_path": self.summary_json_path,
"summary_md_path": self.summary_md_path,
}