ocr-projects-archive / ocr-tech /tools /analyze_page_reading_order_geometry.py
prosperious's picture
Add 1000 files (ocr-tech/tests/test_audit_limbu_book_missed_line_review_coverage.py ...)
d82db76 verified
Raw History Blame Contribute Delete
8.34 kB
#!/usr/bin/env python3
"""Analyze reference-order geometry chunks from page OCR line alignment."""
from __future__ import annotations
import argparse
import csv
import json
import re
import statistics
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
from ocrtech.manifest import sha256_file
from ocrtech.schemas import BBox
REPORT_JSON = "reading-order-geometry-analysis.json"
CHUNKS_TSV = "reading-order-geometry-chunks.tsv"
JUMPS_TSV = "reading-order-geometry-jumps.tsv"
def _read_bbox(value: str, *, context: str) -> BBox:
try:
payload = json.loads(value)
except json.JSONDecodeError as exc:
raise ValueError(f"{context}: bbox is not valid JSON: {value!r}") from exc
try:
bbox = BBox.from_any(payload)
except Exception as exc:
raise ValueError(f"{context}: bbox is invalid: {value!r}") from exc
if bbox.w <= 0 or bbox.h <= 0:
raise ValueError(f"{context}: bbox must have positive width/height: {bbox.to_list()}")
return bbox
def _line_number(line_id: str) -> int:
match = re.search(r"(\d+)$", line_id)
if match is None:
raise ValueError(f"reference_line_id does not end in a numeric order: {line_id!r}")
return int(match.group(1))
def _read_matched_rows(path: Path) -> list[dict[str, Any]]:
try:
handle = path.open("r", encoding="utf-8", newline="")
except FileNotFoundError as exc:
raise ValueError(f"line alignment CSV is missing: {path}") from exc
rows: list[dict[str, Any]] = []
with handle:
reader = csv.DictReader(handle)
if reader.fieldnames is None:
raise ValueError(f"line alignment CSV has no header: {path}")
for row_number, row in enumerate(reader, start=2):
if row.get("kind") != "matched":
continue
reference_line_id = str(row.get("reference_line_id") or "").strip()
predicted_line_id = str(row.get("predicted_line_id") or "").strip()
if not reference_line_id or not predicted_line_id:
continue
bbox = _read_bbox(str(row.get("reference_bbox") or ""), context=f"{path}:{row_number} reference")
rows.append(
{
"reference_line_id": reference_line_id,
"reference_order": _line_number(reference_line_id),
"predicted_line_id": predicted_line_id,
"page_index": int(row.get("page_index") or 0),
"x": bbox.x,
"y": bbox.y,
"w": bbox.w,
"h": bbox.h,
"cx": bbox.x + bbox.w / 2.0,
"cy": bbox.y + bbox.h / 2.0,
}
)
rows.sort(key=lambda item: (item["page_index"], item["reference_order"]))
return rows
def _is_jump(previous: dict[str, Any], current: dict[str, Any], *, min_dx: float, min_upward_dy: float, min_downward_dy: float) -> bool:
dx = float(current["cx"]) - float(previous["cx"])
dy = float(current["y"]) - float(previous["y"])
return abs(dx) > min_dx or dy < -min_upward_dy or dy > min_downward_dy
def _chunk_summary(chunk: list[dict[str, Any]], *, chunk_index: int) -> dict[str, Any]:
return {
"chunk_index": chunk_index,
"start_reference_line_id": chunk[0]["reference_line_id"],
"end_reference_line_id": chunk[-1]["reference_line_id"],
"start_reference_order": chunk[0]["reference_order"],
"end_reference_order": chunk[-1]["reference_order"],
"line_count": len(chunk),
"cx_median": statistics.median(float(row["cx"]) for row in chunk),
"x_min": min(float(row["x"]) for row in chunk),
"x_max": max(float(row["x"]) + float(row["w"]) for row in chunk),
"y_min": min(float(row["y"]) for row in chunk),
"y_max": max(float(row["y"]) for row in chunk),
"width_median": statistics.median(float(row["w"]) for row in chunk),
"predicted_line_ids": [row["predicted_line_id"] for row in chunk],
}
def analyze(args: argparse.Namespace) -> dict[str, Any]:
alignment_path = Path(args.line_alignment)
out = Path(args.out)
out.mkdir(parents=True, exist_ok=True)
rows = _read_matched_rows(alignment_path)
chunks: list[list[dict[str, Any]]] = []
jumps: list[dict[str, Any]] = []
current: list[dict[str, Any]] = []
for row in rows:
if current and _is_jump(
current[-1],
row,
min_dx=float(args.min_dx),
min_upward_dy=float(args.min_upward_dy),
min_downward_dy=float(args.min_downward_dy),
):
previous = current[-1]
jumps.append(
{
"from_reference_line_id": previous["reference_line_id"],
"to_reference_line_id": row["reference_line_id"],
"dx": float(row["cx"]) - float(previous["cx"]),
"dy": float(row["y"]) - float(previous["y"]),
"from_cx": previous["cx"],
"from_y": previous["y"],
"to_cx": row["cx"],
"to_y": row["y"],
}
)
chunks.append(current)
current = []
current.append(row)
if current:
chunks.append(current)
chunk_summaries = [_chunk_summary(chunk, chunk_index=index) for index, chunk in enumerate(chunks, start=1)]
chunks_path = out / CHUNKS_TSV
with chunks_path.open("w", encoding="utf-8", newline="") as handle:
fields = [
"chunk_index",
"start_reference_line_id",
"end_reference_line_id",
"line_count",
"cx_median",
"x_min",
"x_max",
"y_min",
"y_max",
"width_median",
]
writer = csv.DictWriter(handle, fieldnames=fields, delimiter="\t")
writer.writeheader()
for chunk in chunk_summaries:
writer.writerow({field: chunk[field] for field in fields})
jumps_path = out / JUMPS_TSV
with jumps_path.open("w", encoding="utf-8", newline="") as handle:
fields = ["from_reference_line_id", "to_reference_line_id", "dx", "dy", "from_cx", "from_y", "to_cx", "to_y"]
writer = csv.DictWriter(handle, fieldnames=fields, delimiter="\t")
writer.writeheader()
for jump in jumps:
writer.writerow(jump)
report_path = out / REPORT_JSON
report = {
"tool": "tools/analyze_page_reading_order_geometry.py",
"created_at_utc": datetime.now(UTC).isoformat(),
"line_alignment": str(alignment_path),
"line_alignment_sha256": sha256_file(alignment_path),
"output_dir": str(out),
"matched_line_count": len(rows),
"chunk_count": len(chunk_summaries),
"jump_count": len(jumps),
"config": {
"min_dx": float(args.min_dx),
"min_upward_dy": float(args.min_upward_dy),
"min_downward_dy": float(args.min_downward_dy),
},
"chunks": chunk_summaries,
"jumps": jumps,
"chunks_tsv": str(chunks_path),
"chunks_tsv_sha256": sha256_file(chunks_path),
"jumps_tsv": str(jumps_path),
"jumps_tsv_sha256": sha256_file(jumps_path),
"claim_boundary": "diagnostic geometry analysis of eval alignment; use to design non-oracle reading-order policy, not as OCR output",
}
report_path.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
return report
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--line-alignment", type=Path, required=True)
parser.add_argument("--out", type=Path, required=True)
parser.add_argument("--min-dx", type=float, default=350.0)
parser.add_argument("--min-upward-dy", type=float, default=200.0)
parser.add_argument("--min-downward-dy", type=float, default=120.0)
args = parser.parse_args()
report = analyze(args)
print(str(Path(report["output_dir"]) / REPORT_JSON))
print(f"chunks={report['chunk_count']} jumps={report['jump_count']} matched={report['matched_line_count']}")
return 0
if __name__ == "__main__":
raise SystemExit(main())