#!/usr/bin/env python3 """Assemble adjacent OCR fragments into page-line documents. This tool is for non-oracle page-assembly diagnostics. It does not read reference text or reference boxes and does not run OCR. It merges existing OCR line records using only live OCR output text, confidence, metadata, and geometry, preserving source line ids in metadata so the transformation can be audited. """ from __future__ import annotations import argparse import json import platform from datetime import datetime, timezone from pathlib import Path from typing import Any from ocrtech.manifest import load_manifest, sha256_file from ocrtech.schemas import BBox, Block, Document, Page, TextLine, union_bbox def _document_path(documents_dir: Path, sample_id: str) -> Path: return documents_dir / sample_id / "document.json" def _assembly_parameters(args: argparse.Namespace) -> dict[str, Any]: return { "y_tolerance": args.y_tolerance, "max_gap": args.max_gap, "max_horizontal_overlap": args.max_horizontal_overlap, "max_merged_width": args.max_merged_width, "max_merged_width_ratio": args.max_merged_width_ratio, "max_component_width": args.max_component_width, "max_component_width_ratio": args.max_component_width_ratio, "min_vertical_overlap_ratio": args.min_vertical_overlap_ratio, "joiner": args.joiner, "require_text_structural_role": args.require_text_structural_role, } def _center_y(line: TextLine) -> float: return line.bbox.y + (line.bbox.h / 2) def _vertical_overlap_ratio(first: TextLine, second: TextLine) -> float: overlap = min(first.bbox.bottom, second.bbox.bottom) - max(first.bbox.y, second.bbox.y) if overlap <= 0: return 0.0 denom = min(first.bbox.h, second.bbox.h) return overlap / denom if denom > 0 else 0.0 def _structural_role(line: TextLine) -> str: return str((line.metadata or {}).get("structural_role") or "text") def _source_line_ids(line: TextLine) -> list[str]: assembled = (line.metadata or {}).get("line_assembly") if isinstance(assembled, dict): raw = assembled.get("source_line_ids") if isinstance(raw, list) and all(isinstance(item, str) for item in raw): return raw return [line.line_id] if line.line_id else [] def _line_width_ok(line: TextLine, page: Page, args: argparse.Namespace) -> bool: if args.max_component_width is not None and line.bbox.w > args.max_component_width: return False if args.max_component_width_ratio is not None: if not page.width: return False if line.bbox.w / page.width > args.max_component_width_ratio: return False return True def _merged_width_ok(width: float, page: Page, args: argparse.Namespace) -> bool: if args.max_merged_width is not None and width > args.max_merged_width: return False if args.max_merged_width_ratio is not None: if not page.width: return False if width / page.width > args.max_merged_width_ratio: return False return True def _can_merge(first: TextLine, second: TextLine, page: Page, args: argparse.Namespace) -> tuple[bool, list[str]]: reasons: list[str] = [] if args.require_text_structural_role: if _structural_role(first) != "text" or _structural_role(second) != "text": reasons.append("non_text_structural_role") if not first.text.strip() or not second.text.strip(): reasons.append("empty_text") if second.bbox.x < first.bbox.x: reasons.append("not_left_to_right") gap = second.bbox.x - first.bbox.right if gap < 0: overlap = abs(gap) if overlap > args.max_horizontal_overlap: reasons.append("horizontal_overlap") elif gap > args.max_gap: reasons.append("gap_above_max") center_delta = abs(_center_y(first) - _center_y(second)) overlap_ratio = _vertical_overlap_ratio(first, second) if center_delta > args.y_tolerance and overlap_ratio < args.min_vertical_overlap_ratio: reasons.append("row_mismatch") if not _line_width_ok(first, page, args) or not _line_width_ok(second, page, args): reasons.append("component_width_above_max") merged_width = max(first.bbox.right, second.bbox.right) - min(first.bbox.x, second.bbox.x) if not _merged_width_ok(merged_width, page, args): reasons.append("merged_width_above_max") return not reasons, reasons def _merge_group(lines: list[TextLine], *, line_id: str, page_index: int, order: int, args: argparse.Namespace) -> TextLine: bbox = union_bbox([line.bbox for line in lines]) confidences = [line.confidence for line in lines if line.confidence is not None] confidence = min(confidences) if confidences else None text = args.joiner.join(line.text.strip() for line in lines if line.text.strip()) metadata = dict(lines[0].metadata or {}) metadata["line_assembly"] = { "tool": "tools/assemble_page_ocr_lines.py", "assembled": len(lines) > 1, "source_line_ids": [line_id for line in lines for line_id in _source_line_ids(line)], "source_texts": [line.text for line in lines], "source_bboxes": [line.bbox.to_list() for line in lines], "source_confidences": [line.confidence for line in lines], "order": order, "joiner": args.joiner, } return TextLine( text=text, bbox=bbox, confidence=confidence, page_index=page_index, line_id=line_id, metadata=metadata, ) def _assemble_page(page: Page, args: argparse.Namespace) -> tuple[Page, dict[str, Any], list[dict[str, Any]]]: assembled_lines: list[TextLine] = [] line_reports: list[dict[str, Any]] = [] original_block_order_by_line_id: dict[str, tuple[int, str]] = {} original_block_by_line_id: dict[str, Block] = {} for block in sorted(page.blocks, key=lambda item: item.order): for line_id in block.line_ids: if line_id not in original_block_order_by_line_id: original_block_order_by_line_id[line_id] = (block.order, block.block_id) original_block_by_line_id[line_id] = block index = 0 while index < len(page.text_lines): group = [page.text_lines[index]] blocked_reasons: list[str] = [] while index + 1 < len(page.text_lines): can_merge, reasons = _can_merge(group[-1], page.text_lines[index + 1], page, args) if not can_merge: blocked_reasons = reasons break group.append(page.text_lines[index + 1]) index += 1 output_id = group[0].line_id or f"line-{len(assembled_lines) + 1:04d}" if len(group) > 1: output_id = f"assembled-{len(assembled_lines) + 1:04d}" assembled = _merge_group(group, line_id=output_id, page_index=page.page_index, order=len(assembled_lines), args=args) assembled_lines.append(assembled) line_reports.append( { "page_index": page.page_index, "output_line_id": assembled.line_id, "assembled": len(group) > 1, "source_line_ids": [line_id for line in group for line_id in _source_line_ids(line)], "source_texts": [line.text for line in group], "output_text": assembled.text, "output_bbox": assembled.bbox.to_list(), "blocked_next_reasons": blocked_reasons, } ) index += 1 block_rows: list[tuple[tuple[int, str, int], Block]] = [] for line_index, line in enumerate(assembled_lines): source_line_ids = _source_line_ids(line) order_candidates = [ original_block_order_by_line_id[line_id] for line_id in source_line_ids if line_id in original_block_order_by_line_id ] first_source_block = next( (original_block_by_line_id[line_id] for line_id in source_line_ids if line_id in original_block_by_line_id), None, ) sort_key = (*min(order_candidates), line_index) if order_candidates else (line_index, "", line_index) block_type = first_source_block.block_type if first_source_block and first_source_block.block_type in {"text", "title"} else "text" block_metadata = dict(first_source_block.metadata or {}) if first_source_block else {} block_metadata["line_assembly"] = line.metadata.get("line_assembly", {}) block_rows.append( ( sort_key, Block( block_id=f"assembled-block-{line_index + 1:04d}", block_type=block_type, page_index=page.page_index, bbox=line.bbox, order=line_index, text=line.text, confidence=line.confidence, line_ids=[str(line.line_id)] if line.line_id else [], metadata=block_metadata, ), ) ) blocks = [] for order, (_sort_key, block) in enumerate(sorted(block_rows, key=lambda item: item[0])): block.order = order blocks.append(block) metadata = dict(page.metadata or {}) metadata["ocr_line_assembly"] = _assembly_parameters(args) return ( Page( page_index=page.page_index, width=page.width, height=page.height, text_lines=assembled_lines, blocks=blocks, metadata=metadata, ), { "page_index": page.page_index, "source_line_count": len(page.text_lines), "assembled_line_count": len(assembled_lines), "merged_group_count": sum(1 for row in line_reports if row["assembled"]), "merged_source_line_count": sum(len(row["source_line_ids"]) for row in line_reports if row["assembled"]), }, line_reports, ) def _assemble_document(source_path: Path, args: argparse.Namespace) -> tuple[Document, dict[str, Any], list[dict[str, Any]]]: payload = json.loads(source_path.read_text(encoding="utf-8")) source = Document.from_dict(payload) pages: list[Page] = [] page_reports: list[dict[str, Any]] = [] line_reports: list[dict[str, Any]] = [] for page in source.pages: assembled_page, page_report, page_line_reports = _assemble_page(page, args) pages.append(assembled_page) page_reports.append(page_report) line_reports.extend(page_line_reports) source_metadata = dict(source.metadata or {}) source_metadata["ocr_line_assembly_source_document"] = str(source_path) source_metadata["ocr_line_assembly_source_document_sha256"] = sha256_file(source_path) source_metadata["ocr_line_assembly"] = _assembly_parameters(args) source_metadata["line_count"] = sum(len(page.text_lines) for page in pages) return ( Document( source_path=source.source_path, pages=pages, tables=source.tables, figures=source.figures, metadata=source_metadata, ), { "source_document": str(source_path), "source_document_sha256": sha256_file(source_path), "source_line_count": sum(row["source_line_count"] for row in page_reports), "assembled_line_count": sum(row["assembled_line_count"] for row in page_reports), "merged_group_count": sum(row["merged_group_count"] for row in page_reports), "merged_source_line_count": sum(row["merged_source_line_count"] for row in page_reports), "pages": page_reports, }, line_reports, ) def assemble_documents(args: argparse.Namespace) -> dict[str, Any]: if args.y_tolerance < 0: raise ValueError(f"--y-tolerance must be non-negative, got {args.y_tolerance}") if args.max_gap < 0: raise ValueError(f"--max-gap must be non-negative, got {args.max_gap}") if args.max_horizontal_overlap < 0: raise ValueError(f"--max-horizontal-overlap must be non-negative, got {args.max_horizontal_overlap}") for name in ("max_merged_width", "max_component_width"): value = getattr(args, name) if value is not None and value <= 0: raise ValueError(f"--{name.replace('_', '-')} must be positive, got {value}") for name in ("max_merged_width_ratio", "max_component_width_ratio", "min_vertical_overlap_ratio"): value = getattr(args, name) if value is not None and not 0 <= value <= 1: raise ValueError(f"--{name.replace('_', '-')} must be between 0 and 1, got {value}") entries = load_manifest(args.eval_manifest) if not entries: raise ValueError(f"eval manifest is empty: {args.eval_manifest}") args.out.mkdir(parents=True, exist_ok=True) samples: list[dict[str, Any]] = [] all_line_reports: list[dict[str, Any]] = [] for entry in entries: source_path = _document_path(args.documents_dir, entry.sample_id) if not source_path.exists(): raise FileNotFoundError(f"{entry.sample_id}: source document missing: {source_path}") assembled, sample_report, line_reports = _assemble_document(source_path, args) sample_dir = args.out / entry.sample_id sample_dir.mkdir(parents=True, exist_ok=True) output_path = sample_dir / "document.json" output_path.write_text(json.dumps(assembled.to_dict(), ensure_ascii=False, indent=2) + "\n", encoding="utf-8") sample_report.update( { "sample_id": entry.sample_id, "document_path": str(output_path), "document_sha256": sha256_file(output_path), } ) for row in line_reports: row["sample_id"] = entry.sample_id samples.append(sample_report) all_line_reports.extend(line_reports) run = { "created_at": datetime.now(timezone.utc).isoformat(), "machine": platform.node(), "tool": "tools/assemble_page_ocr_lines.py", "eval_manifest": str(args.eval_manifest), "eval_manifest_sha256": sha256_file(args.eval_manifest), "source_documents_dir": str(args.documents_dir), "documents_dir": str(args.out), "assembly": _assembly_parameters(args), "sample_count": len(samples), "source_line_count": sum(row["source_line_count"] for row in samples), "assembled_line_count": sum(row["assembled_line_count"] for row in samples), "merged_group_count": sum(row["merged_group_count"] for row in samples), "merged_source_line_count": sum(row["merged_source_line_count"] for row in samples), "samples": samples, } run_path = args.out / "ocr-line-assembly-run.json" line_report_path = args.out / "ocr-line-assembly-lines.jsonl" report_path = args.out / "README.md" run["run_path"] = str(run_path) run["line_report_path"] = str(line_report_path) run["report_path"] = str(report_path) line_report_path.write_text( "".join(json.dumps(row, ensure_ascii=False, sort_keys=True) + "\n" for row in all_line_reports), encoding="utf-8", ) report_path.write_text( "\n".join( [ "# OCR-Line-Assembled Page OCR Documents", "", "Existing page OCR documents were assembled by live OCR line geometry. No OCR recognition or reference lookup was run.", "", f"- eval manifest: `{args.eval_manifest}`", f"- source documents: `{args.documents_dir}`", f"- output documents: `{args.out}`", f"- assembly: `{json.dumps(run['assembly'], sort_keys=True)}`", f"- source lines: {run['source_line_count']}", f"- assembled lines: {run['assembled_line_count']}", f"- merged groups: {run['merged_group_count']}", f"- merged source lines: {run['merged_source_line_count']}", ] ) + "\n", encoding="utf-8", ) run_path.write_text(json.dumps(run, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") return run def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--eval-manifest", type=Path, required=True) parser.add_argument("--documents-dir", type=Path, required=True) parser.add_argument("--out", type=Path, required=True) parser.add_argument("--y-tolerance", type=float, default=12.0) parser.add_argument("--max-gap", type=float, default=120.0) parser.add_argument("--max-horizontal-overlap", type=float, default=0.0) parser.add_argument("--max-merged-width", type=float) parser.add_argument("--max-merged-width-ratio", type=float) parser.add_argument("--max-component-width", type=float) parser.add_argument("--max-component-width-ratio", type=float) parser.add_argument("--min-vertical-overlap-ratio", type=float, default=0.50) parser.add_argument("--joiner", default=" ") parser.add_argument("--require-text-structural-role", action=argparse.BooleanOptionalAction, default=True) args = parser.parse_args(argv) result = assemble_documents(args) print( f"wrote {result['documents_dir']} source={result['source_line_count']} " f"assembled={result['assembled_line_count']} merged_groups={result['merged_group_count']}" ) return 0 if __name__ == "__main__": raise SystemExit(main())