ocr-projects-archive / ocr-tech /tools /assemble_page_ocr_lines.py
prosperious's picture
Add 1000 files (ocr-tech/tests/test_audit_limbu_book_missed_line_review_coverage.py ...)
d82db76 verified
Raw History Blame Contribute Delete
17.7 kB
#!/usr/bin/env python3
"""Assemble adjacent OCR fragments into page-line documents.
This tool is for non-oracle page-assembly diagnostics. It does not read reference
text or reference boxes and does not run OCR. It merges existing OCR line records
using only live OCR output text, confidence, metadata, and geometry, preserving source
line ids in metadata so the transformation can be audited.
"""
from __future__ import annotations
import argparse
import json
import platform
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
from ocrtech.manifest import load_manifest, sha256_file
from ocrtech.schemas import BBox, Block, Document, Page, TextLine, union_bbox
def _document_path(documents_dir: Path, sample_id: str) -> Path:
return documents_dir / sample_id / "document.json"
def _assembly_parameters(args: argparse.Namespace) -> dict[str, Any]:
return {
"y_tolerance": args.y_tolerance,
"max_gap": args.max_gap,
"max_horizontal_overlap": args.max_horizontal_overlap,
"max_merged_width": args.max_merged_width,
"max_merged_width_ratio": args.max_merged_width_ratio,
"max_component_width": args.max_component_width,
"max_component_width_ratio": args.max_component_width_ratio,
"min_vertical_overlap_ratio": args.min_vertical_overlap_ratio,
"joiner": args.joiner,
"require_text_structural_role": args.require_text_structural_role,
}
def _center_y(line: TextLine) -> float:
return line.bbox.y + (line.bbox.h / 2)
def _vertical_overlap_ratio(first: TextLine, second: TextLine) -> float:
overlap = min(first.bbox.bottom, second.bbox.bottom) - max(first.bbox.y, second.bbox.y)
if overlap <= 0:
return 0.0
denom = min(first.bbox.h, second.bbox.h)
return overlap / denom if denom > 0 else 0.0
def _structural_role(line: TextLine) -> str:
return str((line.metadata or {}).get("structural_role") or "text")
def _source_line_ids(line: TextLine) -> list[str]:
assembled = (line.metadata or {}).get("line_assembly")
if isinstance(assembled, dict):
raw = assembled.get("source_line_ids")
if isinstance(raw, list) and all(isinstance(item, str) for item in raw):
return raw
return [line.line_id] if line.line_id else []
def _line_width_ok(line: TextLine, page: Page, args: argparse.Namespace) -> bool:
if args.max_component_width is not None and line.bbox.w > args.max_component_width:
return False
if args.max_component_width_ratio is not None:
if not page.width:
return False
if line.bbox.w / page.width > args.max_component_width_ratio:
return False
return True
def _merged_width_ok(width: float, page: Page, args: argparse.Namespace) -> bool:
if args.max_merged_width is not None and width > args.max_merged_width:
return False
if args.max_merged_width_ratio is not None:
if not page.width:
return False
if width / page.width > args.max_merged_width_ratio:
return False
return True
def _can_merge(first: TextLine, second: TextLine, page: Page, args: argparse.Namespace) -> tuple[bool, list[str]]:
reasons: list[str] = []
if args.require_text_structural_role:
if _structural_role(first) != "text" or _structural_role(second) != "text":
reasons.append("non_text_structural_role")
if not first.text.strip() or not second.text.strip():
reasons.append("empty_text")
if second.bbox.x < first.bbox.x:
reasons.append("not_left_to_right")
gap = second.bbox.x - first.bbox.right
if gap < 0:
overlap = abs(gap)
if overlap > args.max_horizontal_overlap:
reasons.append("horizontal_overlap")
elif gap > args.max_gap:
reasons.append("gap_above_max")
center_delta = abs(_center_y(first) - _center_y(second))
overlap_ratio = _vertical_overlap_ratio(first, second)
if center_delta > args.y_tolerance and overlap_ratio < args.min_vertical_overlap_ratio:
reasons.append("row_mismatch")
if not _line_width_ok(first, page, args) or not _line_width_ok(second, page, args):
reasons.append("component_width_above_max")
merged_width = max(first.bbox.right, second.bbox.right) - min(first.bbox.x, second.bbox.x)
if not _merged_width_ok(merged_width, page, args):
reasons.append("merged_width_above_max")
return not reasons, reasons
def _merge_group(lines: list[TextLine], *, line_id: str, page_index: int, order: int, args: argparse.Namespace) -> TextLine:
bbox = union_bbox([line.bbox for line in lines])
confidences = [line.confidence for line in lines if line.confidence is not None]
confidence = min(confidences) if confidences else None
text = args.joiner.join(line.text.strip() for line in lines if line.text.strip())
metadata = dict(lines[0].metadata or {})
metadata["line_assembly"] = {
"tool": "tools/assemble_page_ocr_lines.py",
"assembled": len(lines) > 1,
"source_line_ids": [line_id for line in lines for line_id in _source_line_ids(line)],
"source_texts": [line.text for line in lines],
"source_bboxes": [line.bbox.to_list() for line in lines],
"source_confidences": [line.confidence for line in lines],
"order": order,
"joiner": args.joiner,
}
return TextLine(
text=text,
bbox=bbox,
confidence=confidence,
page_index=page_index,
line_id=line_id,
metadata=metadata,
)
def _assemble_page(page: Page, args: argparse.Namespace) -> tuple[Page, dict[str, Any], list[dict[str, Any]]]:
assembled_lines: list[TextLine] = []
line_reports: list[dict[str, Any]] = []
original_block_order_by_line_id: dict[str, tuple[int, str]] = {}
original_block_by_line_id: dict[str, Block] = {}
for block in sorted(page.blocks, key=lambda item: item.order):
for line_id in block.line_ids:
if line_id not in original_block_order_by_line_id:
original_block_order_by_line_id[line_id] = (block.order, block.block_id)
original_block_by_line_id[line_id] = block
index = 0
while index < len(page.text_lines):
group = [page.text_lines[index]]
blocked_reasons: list[str] = []
while index + 1 < len(page.text_lines):
can_merge, reasons = _can_merge(group[-1], page.text_lines[index + 1], page, args)
if not can_merge:
blocked_reasons = reasons
break
group.append(page.text_lines[index + 1])
index += 1
output_id = group[0].line_id or f"line-{len(assembled_lines) + 1:04d}"
if len(group) > 1:
output_id = f"assembled-{len(assembled_lines) + 1:04d}"
assembled = _merge_group(group, line_id=output_id, page_index=page.page_index, order=len(assembled_lines), args=args)
assembled_lines.append(assembled)
line_reports.append(
{
"page_index": page.page_index,
"output_line_id": assembled.line_id,
"assembled": len(group) > 1,
"source_line_ids": [line_id for line in group for line_id in _source_line_ids(line)],
"source_texts": [line.text for line in group],
"output_text": assembled.text,
"output_bbox": assembled.bbox.to_list(),
"blocked_next_reasons": blocked_reasons,
}
)
index += 1
block_rows: list[tuple[tuple[int, str, int], Block]] = []
for line_index, line in enumerate(assembled_lines):
source_line_ids = _source_line_ids(line)
order_candidates = [
original_block_order_by_line_id[line_id]
for line_id in source_line_ids
if line_id in original_block_order_by_line_id
]
first_source_block = next(
(original_block_by_line_id[line_id] for line_id in source_line_ids if line_id in original_block_by_line_id),
None,
)
sort_key = (*min(order_candidates), line_index) if order_candidates else (line_index, "", line_index)
block_type = first_source_block.block_type if first_source_block and first_source_block.block_type in {"text", "title"} else "text"
block_metadata = dict(first_source_block.metadata or {}) if first_source_block else {}
block_metadata["line_assembly"] = line.metadata.get("line_assembly", {})
block_rows.append(
(
sort_key,
Block(
block_id=f"assembled-block-{line_index + 1:04d}",
block_type=block_type,
page_index=page.page_index,
bbox=line.bbox,
order=line_index,
text=line.text,
confidence=line.confidence,
line_ids=[str(line.line_id)] if line.line_id else [],
metadata=block_metadata,
),
)
)
blocks = []
for order, (_sort_key, block) in enumerate(sorted(block_rows, key=lambda item: item[0])):
block.order = order
blocks.append(block)
metadata = dict(page.metadata or {})
metadata["ocr_line_assembly"] = _assembly_parameters(args)
return (
Page(
page_index=page.page_index,
width=page.width,
height=page.height,
text_lines=assembled_lines,
blocks=blocks,
metadata=metadata,
),
{
"page_index": page.page_index,
"source_line_count": len(page.text_lines),
"assembled_line_count": len(assembled_lines),
"merged_group_count": sum(1 for row in line_reports if row["assembled"]),
"merged_source_line_count": sum(len(row["source_line_ids"]) for row in line_reports if row["assembled"]),
},
line_reports,
)
def _assemble_document(source_path: Path, args: argparse.Namespace) -> tuple[Document, dict[str, Any], list[dict[str, Any]]]:
payload = json.loads(source_path.read_text(encoding="utf-8"))
source = Document.from_dict(payload)
pages: list[Page] = []
page_reports: list[dict[str, Any]] = []
line_reports: list[dict[str, Any]] = []
for page in source.pages:
assembled_page, page_report, page_line_reports = _assemble_page(page, args)
pages.append(assembled_page)
page_reports.append(page_report)
line_reports.extend(page_line_reports)
source_metadata = dict(source.metadata or {})
source_metadata["ocr_line_assembly_source_document"] = str(source_path)
source_metadata["ocr_line_assembly_source_document_sha256"] = sha256_file(source_path)
source_metadata["ocr_line_assembly"] = _assembly_parameters(args)
source_metadata["line_count"] = sum(len(page.text_lines) for page in pages)
return (
Document(
source_path=source.source_path,
pages=pages,
tables=source.tables,
figures=source.figures,
metadata=source_metadata,
),
{
"source_document": str(source_path),
"source_document_sha256": sha256_file(source_path),
"source_line_count": sum(row["source_line_count"] for row in page_reports),
"assembled_line_count": sum(row["assembled_line_count"] for row in page_reports),
"merged_group_count": sum(row["merged_group_count"] for row in page_reports),
"merged_source_line_count": sum(row["merged_source_line_count"] for row in page_reports),
"pages": page_reports,
},
line_reports,
)
def assemble_documents(args: argparse.Namespace) -> dict[str, Any]:
if args.y_tolerance < 0:
raise ValueError(f"--y-tolerance must be non-negative, got {args.y_tolerance}")
if args.max_gap < 0:
raise ValueError(f"--max-gap must be non-negative, got {args.max_gap}")
if args.max_horizontal_overlap < 0:
raise ValueError(f"--max-horizontal-overlap must be non-negative, got {args.max_horizontal_overlap}")
for name in ("max_merged_width", "max_component_width"):
value = getattr(args, name)
if value is not None and value <= 0:
raise ValueError(f"--{name.replace('_', '-')} must be positive, got {value}")
for name in ("max_merged_width_ratio", "max_component_width_ratio", "min_vertical_overlap_ratio"):
value = getattr(args, name)
if value is not None and not 0 <= value <= 1:
raise ValueError(f"--{name.replace('_', '-')} must be between 0 and 1, got {value}")
entries = load_manifest(args.eval_manifest)
if not entries:
raise ValueError(f"eval manifest is empty: {args.eval_manifest}")
args.out.mkdir(parents=True, exist_ok=True)
samples: list[dict[str, Any]] = []
all_line_reports: list[dict[str, Any]] = []
for entry in entries:
source_path = _document_path(args.documents_dir, entry.sample_id)
if not source_path.exists():
raise FileNotFoundError(f"{entry.sample_id}: source document missing: {source_path}")
assembled, sample_report, line_reports = _assemble_document(source_path, args)
sample_dir = args.out / entry.sample_id
sample_dir.mkdir(parents=True, exist_ok=True)
output_path = sample_dir / "document.json"
output_path.write_text(json.dumps(assembled.to_dict(), ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
sample_report.update(
{
"sample_id": entry.sample_id,
"document_path": str(output_path),
"document_sha256": sha256_file(output_path),
}
)
for row in line_reports:
row["sample_id"] = entry.sample_id
samples.append(sample_report)
all_line_reports.extend(line_reports)
run = {
"created_at": datetime.now(timezone.utc).isoformat(),
"machine": platform.node(),
"tool": "tools/assemble_page_ocr_lines.py",
"eval_manifest": str(args.eval_manifest),
"eval_manifest_sha256": sha256_file(args.eval_manifest),
"source_documents_dir": str(args.documents_dir),
"documents_dir": str(args.out),
"assembly": _assembly_parameters(args),
"sample_count": len(samples),
"source_line_count": sum(row["source_line_count"] for row in samples),
"assembled_line_count": sum(row["assembled_line_count"] for row in samples),
"merged_group_count": sum(row["merged_group_count"] for row in samples),
"merged_source_line_count": sum(row["merged_source_line_count"] for row in samples),
"samples": samples,
}
run_path = args.out / "ocr-line-assembly-run.json"
line_report_path = args.out / "ocr-line-assembly-lines.jsonl"
report_path = args.out / "README.md"
run["run_path"] = str(run_path)
run["line_report_path"] = str(line_report_path)
run["report_path"] = str(report_path)
line_report_path.write_text(
"".join(json.dumps(row, ensure_ascii=False, sort_keys=True) + "\n" for row in all_line_reports),
encoding="utf-8",
)
report_path.write_text(
"\n".join(
[
"# OCR-Line-Assembled Page OCR Documents",
"",
"Existing page OCR documents were assembled by live OCR line geometry. No OCR recognition or reference lookup was run.",
"",
f"- eval manifest: `{args.eval_manifest}`",
f"- source documents: `{args.documents_dir}`",
f"- output documents: `{args.out}`",
f"- assembly: `{json.dumps(run['assembly'], sort_keys=True)}`",
f"- source lines: {run['source_line_count']}",
f"- assembled lines: {run['assembled_line_count']}",
f"- merged groups: {run['merged_group_count']}",
f"- merged source lines: {run['merged_source_line_count']}",
]
)
+ "\n",
encoding="utf-8",
)
run_path.write_text(json.dumps(run, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
return run
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--eval-manifest", type=Path, required=True)
parser.add_argument("--documents-dir", type=Path, required=True)
parser.add_argument("--out", type=Path, required=True)
parser.add_argument("--y-tolerance", type=float, default=12.0)
parser.add_argument("--max-gap", type=float, default=120.0)
parser.add_argument("--max-horizontal-overlap", type=float, default=0.0)
parser.add_argument("--max-merged-width", type=float)
parser.add_argument("--max-merged-width-ratio", type=float)
parser.add_argument("--max-component-width", type=float)
parser.add_argument("--max-component-width-ratio", type=float)
parser.add_argument("--min-vertical-overlap-ratio", type=float, default=0.50)
parser.add_argument("--joiner", default=" ")
parser.add_argument("--require-text-structural-role", action=argparse.BooleanOptionalAction, default=True)
args = parser.parse_args(argv)
result = assemble_documents(args)
print(
f"wrote {result['documents_dir']} source={result['source_line_count']} "
f"assembled={result['assembled_line_count']} merged_groups={result['merged_group_count']}"
)
return 0
if __name__ == "__main__":
raise SystemExit(main())