#!/usr/bin/env python3 """Analyze reference-order geometry chunks from page OCR line alignment.""" from __future__ import annotations import argparse import csv import json import re import statistics from datetime import UTC, datetime from pathlib import Path from typing import Any from ocrtech.manifest import sha256_file from ocrtech.schemas import BBox REPORT_JSON = "reading-order-geometry-analysis.json" CHUNKS_TSV = "reading-order-geometry-chunks.tsv" JUMPS_TSV = "reading-order-geometry-jumps.tsv" def _read_bbox(value: str, *, context: str) -> BBox: try: payload = json.loads(value) except json.JSONDecodeError as exc: raise ValueError(f"{context}: bbox is not valid JSON: {value!r}") from exc try: bbox = BBox.from_any(payload) except Exception as exc: raise ValueError(f"{context}: bbox is invalid: {value!r}") from exc if bbox.w <= 0 or bbox.h <= 0: raise ValueError(f"{context}: bbox must have positive width/height: {bbox.to_list()}") return bbox def _line_number(line_id: str) -> int: match = re.search(r"(\d+)$", line_id) if match is None: raise ValueError(f"reference_line_id does not end in a numeric order: {line_id!r}") return int(match.group(1)) def _read_matched_rows(path: Path) -> list[dict[str, Any]]: try: handle = path.open("r", encoding="utf-8", newline="") except FileNotFoundError as exc: raise ValueError(f"line alignment CSV is missing: {path}") from exc rows: list[dict[str, Any]] = [] with handle: reader = csv.DictReader(handle) if reader.fieldnames is None: raise ValueError(f"line alignment CSV has no header: {path}") for row_number, row in enumerate(reader, start=2): if row.get("kind") != "matched": continue reference_line_id = str(row.get("reference_line_id") or "").strip() predicted_line_id = str(row.get("predicted_line_id") or "").strip() if not reference_line_id or not predicted_line_id: continue bbox = _read_bbox(str(row.get("reference_bbox") or ""), context=f"{path}:{row_number} reference") rows.append( { "reference_line_id": reference_line_id, "reference_order": _line_number(reference_line_id), "predicted_line_id": predicted_line_id, "page_index": int(row.get("page_index") or 0), "x": bbox.x, "y": bbox.y, "w": bbox.w, "h": bbox.h, "cx": bbox.x + bbox.w / 2.0, "cy": bbox.y + bbox.h / 2.0, } ) rows.sort(key=lambda item: (item["page_index"], item["reference_order"])) return rows def _is_jump(previous: dict[str, Any], current: dict[str, Any], *, min_dx: float, min_upward_dy: float, min_downward_dy: float) -> bool: dx = float(current["cx"]) - float(previous["cx"]) dy = float(current["y"]) - float(previous["y"]) return abs(dx) > min_dx or dy < -min_upward_dy or dy > min_downward_dy def _chunk_summary(chunk: list[dict[str, Any]], *, chunk_index: int) -> dict[str, Any]: return { "chunk_index": chunk_index, "start_reference_line_id": chunk[0]["reference_line_id"], "end_reference_line_id": chunk[-1]["reference_line_id"], "start_reference_order": chunk[0]["reference_order"], "end_reference_order": chunk[-1]["reference_order"], "line_count": len(chunk), "cx_median": statistics.median(float(row["cx"]) for row in chunk), "x_min": min(float(row["x"]) for row in chunk), "x_max": max(float(row["x"]) + float(row["w"]) for row in chunk), "y_min": min(float(row["y"]) for row in chunk), "y_max": max(float(row["y"]) for row in chunk), "width_median": statistics.median(float(row["w"]) for row in chunk), "predicted_line_ids": [row["predicted_line_id"] for row in chunk], } def analyze(args: argparse.Namespace) -> dict[str, Any]: alignment_path = Path(args.line_alignment) out = Path(args.out) out.mkdir(parents=True, exist_ok=True) rows = _read_matched_rows(alignment_path) chunks: list[list[dict[str, Any]]] = [] jumps: list[dict[str, Any]] = [] current: list[dict[str, Any]] = [] for row in rows: if current and _is_jump( current[-1], row, min_dx=float(args.min_dx), min_upward_dy=float(args.min_upward_dy), min_downward_dy=float(args.min_downward_dy), ): previous = current[-1] jumps.append( { "from_reference_line_id": previous["reference_line_id"], "to_reference_line_id": row["reference_line_id"], "dx": float(row["cx"]) - float(previous["cx"]), "dy": float(row["y"]) - float(previous["y"]), "from_cx": previous["cx"], "from_y": previous["y"], "to_cx": row["cx"], "to_y": row["y"], } ) chunks.append(current) current = [] current.append(row) if current: chunks.append(current) chunk_summaries = [_chunk_summary(chunk, chunk_index=index) for index, chunk in enumerate(chunks, start=1)] chunks_path = out / CHUNKS_TSV with chunks_path.open("w", encoding="utf-8", newline="") as handle: fields = [ "chunk_index", "start_reference_line_id", "end_reference_line_id", "line_count", "cx_median", "x_min", "x_max", "y_min", "y_max", "width_median", ] writer = csv.DictWriter(handle, fieldnames=fields, delimiter="\t") writer.writeheader() for chunk in chunk_summaries: writer.writerow({field: chunk[field] for field in fields}) jumps_path = out / JUMPS_TSV with jumps_path.open("w", encoding="utf-8", newline="") as handle: fields = ["from_reference_line_id", "to_reference_line_id", "dx", "dy", "from_cx", "from_y", "to_cx", "to_y"] writer = csv.DictWriter(handle, fieldnames=fields, delimiter="\t") writer.writeheader() for jump in jumps: writer.writerow(jump) report_path = out / REPORT_JSON report = { "tool": "tools/analyze_page_reading_order_geometry.py", "created_at_utc": datetime.now(UTC).isoformat(), "line_alignment": str(alignment_path), "line_alignment_sha256": sha256_file(alignment_path), "output_dir": str(out), "matched_line_count": len(rows), "chunk_count": len(chunk_summaries), "jump_count": len(jumps), "config": { "min_dx": float(args.min_dx), "min_upward_dy": float(args.min_upward_dy), "min_downward_dy": float(args.min_downward_dy), }, "chunks": chunk_summaries, "jumps": jumps, "chunks_tsv": str(chunks_path), "chunks_tsv_sha256": sha256_file(chunks_path), "jumps_tsv": str(jumps_path), "jumps_tsv_sha256": sha256_file(jumps_path), "claim_boundary": "diagnostic geometry analysis of eval alignment; use to design non-oracle reading-order policy, not as OCR output", } report_path.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") return report def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--line-alignment", type=Path, required=True) parser.add_argument("--out", type=Path, required=True) parser.add_argument("--min-dx", type=float, default=350.0) parser.add_argument("--min-upward-dy", type=float, default=200.0) parser.add_argument("--min-downward-dy", type=float, default=120.0) args = parser.parse_args() report = analyze(args) print(str(Path(report["output_dir"]) / REPORT_JSON)) print(f"chunks={report['chunk_count']} jumps={report['jump_count']} matched={report['matched_line_count']}") return 0 if __name__ == "__main__": raise SystemExit(main())