Add 1000 files (ocr-tech/tests/test_audit_limbu_book_missed_line_review_coverage.py ...)
d82db76 verified Download ocr-tech/tools/assemble_page_ocr_lines.py from prosperious/ocr-projects-archive: direct link, hf CLI and curl.
- Browser
- Download file 17.7 kB
-
https://huggingface.co/prosperious/ocr-projects-archive/resolve/main/ocr-tech/tools/assemble_page_ocr_lines.py
- Command line
-
hf download hf://prosperious/ocr-projects-archive/ocr-tech/tools/assemble_page_ocr_lines.py
-
curl -L -o assemble_page_ocr_lines.py https://huggingface.co/prosperious/ocr-projects-archive/resolve/main/ocr-tech/tools/assemble_page_ocr_lines.py
17.7 kB
| #!/usr/bin/env python3 | |
| """Assemble adjacent OCR fragments into page-line documents. | |
| This tool is for non-oracle page-assembly diagnostics. It does not read reference | |
| text or reference boxes and does not run OCR. It merges existing OCR line records | |
| using only live OCR output text, confidence, metadata, and geometry, preserving source | |
| line ids in metadata so the transformation can be audited. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import platform | |
| from datetime import datetime, timezone | |
| from pathlib import Path | |
| from typing import Any | |
| from ocrtech.manifest import load_manifest, sha256_file | |
| from ocrtech.schemas import BBox, Block, Document, Page, TextLine, union_bbox | |
| def _document_path(documents_dir: Path, sample_id: str) -> Path: | |
| return documents_dir / sample_id / "document.json" | |
| def _assembly_parameters(args: argparse.Namespace) -> dict[str, Any]: | |
| return { | |
| "y_tolerance": args.y_tolerance, | |
| "max_gap": args.max_gap, | |
| "max_horizontal_overlap": args.max_horizontal_overlap, | |
| "max_merged_width": args.max_merged_width, | |
| "max_merged_width_ratio": args.max_merged_width_ratio, | |
| "max_component_width": args.max_component_width, | |
| "max_component_width_ratio": args.max_component_width_ratio, | |
| "min_vertical_overlap_ratio": args.min_vertical_overlap_ratio, | |
| "joiner": args.joiner, | |
| "require_text_structural_role": args.require_text_structural_role, | |
| } | |
| def _center_y(line: TextLine) -> float: | |
| return line.bbox.y + (line.bbox.h / 2) | |
| def _vertical_overlap_ratio(first: TextLine, second: TextLine) -> float: | |
| overlap = min(first.bbox.bottom, second.bbox.bottom) - max(first.bbox.y, second.bbox.y) | |
| if overlap <= 0: | |
| return 0.0 | |
| denom = min(first.bbox.h, second.bbox.h) | |
| return overlap / denom if denom > 0 else 0.0 | |
| def _structural_role(line: TextLine) -> str: | |
| return str((line.metadata or {}).get("structural_role") or "text") | |
| def _source_line_ids(line: TextLine) -> list[str]: | |
| assembled = (line.metadata or {}).get("line_assembly") | |
| if isinstance(assembled, dict): | |
| raw = assembled.get("source_line_ids") | |
| if isinstance(raw, list) and all(isinstance(item, str) for item in raw): | |
| return raw | |
| return [line.line_id] if line.line_id else [] | |
| def _line_width_ok(line: TextLine, page: Page, args: argparse.Namespace) -> bool: | |
| if args.max_component_width is not None and line.bbox.w > args.max_component_width: | |
| return False | |
| if args.max_component_width_ratio is not None: | |
| if not page.width: | |
| return False | |
| if line.bbox.w / page.width > args.max_component_width_ratio: | |
| return False | |
| return True | |
| def _merged_width_ok(width: float, page: Page, args: argparse.Namespace) -> bool: | |
| if args.max_merged_width is not None and width > args.max_merged_width: | |
| return False | |
| if args.max_merged_width_ratio is not None: | |
| if not page.width: | |
| return False | |
| if width / page.width > args.max_merged_width_ratio: | |
| return False | |
| return True | |
| def _can_merge(first: TextLine, second: TextLine, page: Page, args: argparse.Namespace) -> tuple[bool, list[str]]: | |
| reasons: list[str] = [] | |
| if args.require_text_structural_role: | |
| if _structural_role(first) != "text" or _structural_role(second) != "text": | |
| reasons.append("non_text_structural_role") | |
| if not first.text.strip() or not second.text.strip(): | |
| reasons.append("empty_text") | |
| if second.bbox.x < first.bbox.x: | |
| reasons.append("not_left_to_right") | |
| gap = second.bbox.x - first.bbox.right | |
| if gap < 0: | |
| overlap = abs(gap) | |
| if overlap > args.max_horizontal_overlap: | |
| reasons.append("horizontal_overlap") | |
| elif gap > args.max_gap: | |
| reasons.append("gap_above_max") | |
| center_delta = abs(_center_y(first) - _center_y(second)) | |
| overlap_ratio = _vertical_overlap_ratio(first, second) | |
| if center_delta > args.y_tolerance and overlap_ratio < args.min_vertical_overlap_ratio: | |
| reasons.append("row_mismatch") | |
| if not _line_width_ok(first, page, args) or not _line_width_ok(second, page, args): | |
| reasons.append("component_width_above_max") | |
| merged_width = max(first.bbox.right, second.bbox.right) - min(first.bbox.x, second.bbox.x) | |
| if not _merged_width_ok(merged_width, page, args): | |
| reasons.append("merged_width_above_max") | |
| return not reasons, reasons | |
| def _merge_group(lines: list[TextLine], *, line_id: str, page_index: int, order: int, args: argparse.Namespace) -> TextLine: | |
| bbox = union_bbox([line.bbox for line in lines]) | |
| confidences = [line.confidence for line in lines if line.confidence is not None] | |
| confidence = min(confidences) if confidences else None | |
| text = args.joiner.join(line.text.strip() for line in lines if line.text.strip()) | |
| metadata = dict(lines[0].metadata or {}) | |
| metadata["line_assembly"] = { | |
| "tool": "tools/assemble_page_ocr_lines.py", | |
| "assembled": len(lines) > 1, | |
| "source_line_ids": [line_id for line in lines for line_id in _source_line_ids(line)], | |
| "source_texts": [line.text for line in lines], | |
| "source_bboxes": [line.bbox.to_list() for line in lines], | |
| "source_confidences": [line.confidence for line in lines], | |
| "order": order, | |
| "joiner": args.joiner, | |
| } | |
| return TextLine( | |
| text=text, | |
| bbox=bbox, | |
| confidence=confidence, | |
| page_index=page_index, | |
| line_id=line_id, | |
| metadata=metadata, | |
| ) | |
| def _assemble_page(page: Page, args: argparse.Namespace) -> tuple[Page, dict[str, Any], list[dict[str, Any]]]: | |
| assembled_lines: list[TextLine] = [] | |
| line_reports: list[dict[str, Any]] = [] | |
| original_block_order_by_line_id: dict[str, tuple[int, str]] = {} | |
| original_block_by_line_id: dict[str, Block] = {} | |
| for block in sorted(page.blocks, key=lambda item: item.order): | |
| for line_id in block.line_ids: | |
| if line_id not in original_block_order_by_line_id: | |
| original_block_order_by_line_id[line_id] = (block.order, block.block_id) | |
| original_block_by_line_id[line_id] = block | |
| index = 0 | |
| while index < len(page.text_lines): | |
| group = [page.text_lines[index]] | |
| blocked_reasons: list[str] = [] | |
| while index + 1 < len(page.text_lines): | |
| can_merge, reasons = _can_merge(group[-1], page.text_lines[index + 1], page, args) | |
| if not can_merge: | |
| blocked_reasons = reasons | |
| break | |
| group.append(page.text_lines[index + 1]) | |
| index += 1 | |
| output_id = group[0].line_id or f"line-{len(assembled_lines) + 1:04d}" | |
| if len(group) > 1: | |
| output_id = f"assembled-{len(assembled_lines) + 1:04d}" | |
| assembled = _merge_group(group, line_id=output_id, page_index=page.page_index, order=len(assembled_lines), args=args) | |
| assembled_lines.append(assembled) | |
| line_reports.append( | |
| { | |
| "page_index": page.page_index, | |
| "output_line_id": assembled.line_id, | |
| "assembled": len(group) > 1, | |
| "source_line_ids": [line_id for line in group for line_id in _source_line_ids(line)], | |
| "source_texts": [line.text for line in group], | |
| "output_text": assembled.text, | |
| "output_bbox": assembled.bbox.to_list(), | |
| "blocked_next_reasons": blocked_reasons, | |
| } | |
| ) | |
| index += 1 | |
| block_rows: list[tuple[tuple[int, str, int], Block]] = [] | |
| for line_index, line in enumerate(assembled_lines): | |
| source_line_ids = _source_line_ids(line) | |
| order_candidates = [ | |
| original_block_order_by_line_id[line_id] | |
| for line_id in source_line_ids | |
| if line_id in original_block_order_by_line_id | |
| ] | |
| first_source_block = next( | |
| (original_block_by_line_id[line_id] for line_id in source_line_ids if line_id in original_block_by_line_id), | |
| None, | |
| ) | |
| sort_key = (*min(order_candidates), line_index) if order_candidates else (line_index, "", line_index) | |
| block_type = first_source_block.block_type if first_source_block and first_source_block.block_type in {"text", "title"} else "text" | |
| block_metadata = dict(first_source_block.metadata or {}) if first_source_block else {} | |
| block_metadata["line_assembly"] = line.metadata.get("line_assembly", {}) | |
| block_rows.append( | |
| ( | |
| sort_key, | |
| Block( | |
| block_id=f"assembled-block-{line_index + 1:04d}", | |
| block_type=block_type, | |
| page_index=page.page_index, | |
| bbox=line.bbox, | |
| order=line_index, | |
| text=line.text, | |
| confidence=line.confidence, | |
| line_ids=[str(line.line_id)] if line.line_id else [], | |
| metadata=block_metadata, | |
| ), | |
| ) | |
| ) | |
| blocks = [] | |
| for order, (_sort_key, block) in enumerate(sorted(block_rows, key=lambda item: item[0])): | |
| block.order = order | |
| blocks.append(block) | |
| metadata = dict(page.metadata or {}) | |
| metadata["ocr_line_assembly"] = _assembly_parameters(args) | |
| return ( | |
| Page( | |
| page_index=page.page_index, | |
| width=page.width, | |
| height=page.height, | |
| text_lines=assembled_lines, | |
| blocks=blocks, | |
| metadata=metadata, | |
| ), | |
| { | |
| "page_index": page.page_index, | |
| "source_line_count": len(page.text_lines), | |
| "assembled_line_count": len(assembled_lines), | |
| "merged_group_count": sum(1 for row in line_reports if row["assembled"]), | |
| "merged_source_line_count": sum(len(row["source_line_ids"]) for row in line_reports if row["assembled"]), | |
| }, | |
| line_reports, | |
| ) | |
| def _assemble_document(source_path: Path, args: argparse.Namespace) -> tuple[Document, dict[str, Any], list[dict[str, Any]]]: | |
| payload = json.loads(source_path.read_text(encoding="utf-8")) | |
| source = Document.from_dict(payload) | |
| pages: list[Page] = [] | |
| page_reports: list[dict[str, Any]] = [] | |
| line_reports: list[dict[str, Any]] = [] | |
| for page in source.pages: | |
| assembled_page, page_report, page_line_reports = _assemble_page(page, args) | |
| pages.append(assembled_page) | |
| page_reports.append(page_report) | |
| line_reports.extend(page_line_reports) | |
| source_metadata = dict(source.metadata or {}) | |
| source_metadata["ocr_line_assembly_source_document"] = str(source_path) | |
| source_metadata["ocr_line_assembly_source_document_sha256"] = sha256_file(source_path) | |
| source_metadata["ocr_line_assembly"] = _assembly_parameters(args) | |
| source_metadata["line_count"] = sum(len(page.text_lines) for page in pages) | |
| return ( | |
| Document( | |
| source_path=source.source_path, | |
| pages=pages, | |
| tables=source.tables, | |
| figures=source.figures, | |
| metadata=source_metadata, | |
| ), | |
| { | |
| "source_document": str(source_path), | |
| "source_document_sha256": sha256_file(source_path), | |
| "source_line_count": sum(row["source_line_count"] for row in page_reports), | |
| "assembled_line_count": sum(row["assembled_line_count"] for row in page_reports), | |
| "merged_group_count": sum(row["merged_group_count"] for row in page_reports), | |
| "merged_source_line_count": sum(row["merged_source_line_count"] for row in page_reports), | |
| "pages": page_reports, | |
| }, | |
| line_reports, | |
| ) | |
| def assemble_documents(args: argparse.Namespace) -> dict[str, Any]: | |
| if args.y_tolerance < 0: | |
| raise ValueError(f"--y-tolerance must be non-negative, got {args.y_tolerance}") | |
| if args.max_gap < 0: | |
| raise ValueError(f"--max-gap must be non-negative, got {args.max_gap}") | |
| if args.max_horizontal_overlap < 0: | |
| raise ValueError(f"--max-horizontal-overlap must be non-negative, got {args.max_horizontal_overlap}") | |
| for name in ("max_merged_width", "max_component_width"): | |
| value = getattr(args, name) | |
| if value is not None and value <= 0: | |
| raise ValueError(f"--{name.replace('_', '-')} must be positive, got {value}") | |
| for name in ("max_merged_width_ratio", "max_component_width_ratio", "min_vertical_overlap_ratio"): | |
| value = getattr(args, name) | |
| if value is not None and not 0 <= value <= 1: | |
| raise ValueError(f"--{name.replace('_', '-')} must be between 0 and 1, got {value}") | |
| entries = load_manifest(args.eval_manifest) | |
| if not entries: | |
| raise ValueError(f"eval manifest is empty: {args.eval_manifest}") | |
| args.out.mkdir(parents=True, exist_ok=True) | |
| samples: list[dict[str, Any]] = [] | |
| all_line_reports: list[dict[str, Any]] = [] | |
| for entry in entries: | |
| source_path = _document_path(args.documents_dir, entry.sample_id) | |
| if not source_path.exists(): | |
| raise FileNotFoundError(f"{entry.sample_id}: source document missing: {source_path}") | |
| assembled, sample_report, line_reports = _assemble_document(source_path, args) | |
| sample_dir = args.out / entry.sample_id | |
| sample_dir.mkdir(parents=True, exist_ok=True) | |
| output_path = sample_dir / "document.json" | |
| output_path.write_text(json.dumps(assembled.to_dict(), ensure_ascii=False, indent=2) + "\n", encoding="utf-8") | |
| sample_report.update( | |
| { | |
| "sample_id": entry.sample_id, | |
| "document_path": str(output_path), | |
| "document_sha256": sha256_file(output_path), | |
| } | |
| ) | |
| for row in line_reports: | |
| row["sample_id"] = entry.sample_id | |
| samples.append(sample_report) | |
| all_line_reports.extend(line_reports) | |
| run = { | |
| "created_at": datetime.now(timezone.utc).isoformat(), | |
| "machine": platform.node(), | |
| "tool": "tools/assemble_page_ocr_lines.py", | |
| "eval_manifest": str(args.eval_manifest), | |
| "eval_manifest_sha256": sha256_file(args.eval_manifest), | |
| "source_documents_dir": str(args.documents_dir), | |
| "documents_dir": str(args.out), | |
| "assembly": _assembly_parameters(args), | |
| "sample_count": len(samples), | |
| "source_line_count": sum(row["source_line_count"] for row in samples), | |
| "assembled_line_count": sum(row["assembled_line_count"] for row in samples), | |
| "merged_group_count": sum(row["merged_group_count"] for row in samples), | |
| "merged_source_line_count": sum(row["merged_source_line_count"] for row in samples), | |
| "samples": samples, | |
| } | |
| run_path = args.out / "ocr-line-assembly-run.json" | |
| line_report_path = args.out / "ocr-line-assembly-lines.jsonl" | |
| report_path = args.out / "README.md" | |
| run["run_path"] = str(run_path) | |
| run["line_report_path"] = str(line_report_path) | |
| run["report_path"] = str(report_path) | |
| line_report_path.write_text( | |
| "".join(json.dumps(row, ensure_ascii=False, sort_keys=True) + "\n" for row in all_line_reports), | |
| encoding="utf-8", | |
| ) | |
| report_path.write_text( | |
| "\n".join( | |
| [ | |
| "# OCR-Line-Assembled Page OCR Documents", | |
| "", | |
| "Existing page OCR documents were assembled by live OCR line geometry. No OCR recognition or reference lookup was run.", | |
| "", | |
| f"- eval manifest: `{args.eval_manifest}`", | |
| f"- source documents: `{args.documents_dir}`", | |
| f"- output documents: `{args.out}`", | |
| f"- assembly: `{json.dumps(run['assembly'], sort_keys=True)}`", | |
| f"- source lines: {run['source_line_count']}", | |
| f"- assembled lines: {run['assembled_line_count']}", | |
| f"- merged groups: {run['merged_group_count']}", | |
| f"- merged source lines: {run['merged_source_line_count']}", | |
| ] | |
| ) | |
| + "\n", | |
| encoding="utf-8", | |
| ) | |
| run_path.write_text(json.dumps(run, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") | |
| return run | |
| def main(argv: list[str] | None = None) -> int: | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument("--eval-manifest", type=Path, required=True) | |
| parser.add_argument("--documents-dir", type=Path, required=True) | |
| parser.add_argument("--out", type=Path, required=True) | |
| parser.add_argument("--y-tolerance", type=float, default=12.0) | |
| parser.add_argument("--max-gap", type=float, default=120.0) | |
| parser.add_argument("--max-horizontal-overlap", type=float, default=0.0) | |
| parser.add_argument("--max-merged-width", type=float) | |
| parser.add_argument("--max-merged-width-ratio", type=float) | |
| parser.add_argument("--max-component-width", type=float) | |
| parser.add_argument("--max-component-width-ratio", type=float) | |
| parser.add_argument("--min-vertical-overlap-ratio", type=float, default=0.50) | |
| parser.add_argument("--joiner", default=" ") | |
| parser.add_argument("--require-text-structural-role", action=argparse.BooleanOptionalAction, default=True) | |
| args = parser.parse_args(argv) | |
| result = assemble_documents(args) | |
| print( | |
| f"wrote {result['documents_dir']} source={result['source_line_count']} " | |
| f"assembled={result['assembled_line_count']} merged_groups={result['merged_group_count']}" | |
| ) | |
| return 0 | |
| if __name__ == "__main__": | |
| raise SystemExit(main()) | |