Add 1000 files (ocr-tech/tests/test_audit_limbu_book_missed_line_review_coverage.py ...)
d82db76 verified Download ocr-tech/tools/analyze_page_reading_order_geometry.py from prosperious/ocr-projects-archive: direct link, hf CLI and curl.
- Browser
- Download file 8.34 kB
-
https://huggingface.co/prosperious/ocr-projects-archive/resolve/main/ocr-tech/tools/analyze_page_reading_order_geometry.py
- Command line
-
hf download hf://prosperious/ocr-projects-archive/ocr-tech/tools/analyze_page_reading_order_geometry.py
-
curl -L -o analyze_page_reading_order_geometry.py https://huggingface.co/prosperious/ocr-projects-archive/resolve/main/ocr-tech/tools/analyze_page_reading_order_geometry.py
8.34 kB
| #!/usr/bin/env python3 | |
| """Analyze reference-order geometry chunks from page OCR line alignment.""" | |
| from __future__ import annotations | |
| import argparse | |
| import csv | |
| import json | |
| import re | |
| import statistics | |
| from datetime import UTC, datetime | |
| from pathlib import Path | |
| from typing import Any | |
| from ocrtech.manifest import sha256_file | |
| from ocrtech.schemas import BBox | |
| REPORT_JSON = "reading-order-geometry-analysis.json" | |
| CHUNKS_TSV = "reading-order-geometry-chunks.tsv" | |
| JUMPS_TSV = "reading-order-geometry-jumps.tsv" | |
| def _read_bbox(value: str, *, context: str) -> BBox: | |
| try: | |
| payload = json.loads(value) | |
| except json.JSONDecodeError as exc: | |
| raise ValueError(f"{context}: bbox is not valid JSON: {value!r}") from exc | |
| try: | |
| bbox = BBox.from_any(payload) | |
| except Exception as exc: | |
| raise ValueError(f"{context}: bbox is invalid: {value!r}") from exc | |
| if bbox.w <= 0 or bbox.h <= 0: | |
| raise ValueError(f"{context}: bbox must have positive width/height: {bbox.to_list()}") | |
| return bbox | |
| def _line_number(line_id: str) -> int: | |
| match = re.search(r"(\d+)$", line_id) | |
| if match is None: | |
| raise ValueError(f"reference_line_id does not end in a numeric order: {line_id!r}") | |
| return int(match.group(1)) | |
| def _read_matched_rows(path: Path) -> list[dict[str, Any]]: | |
| try: | |
| handle = path.open("r", encoding="utf-8", newline="") | |
| except FileNotFoundError as exc: | |
| raise ValueError(f"line alignment CSV is missing: {path}") from exc | |
| rows: list[dict[str, Any]] = [] | |
| with handle: | |
| reader = csv.DictReader(handle) | |
| if reader.fieldnames is None: | |
| raise ValueError(f"line alignment CSV has no header: {path}") | |
| for row_number, row in enumerate(reader, start=2): | |
| if row.get("kind") != "matched": | |
| continue | |
| reference_line_id = str(row.get("reference_line_id") or "").strip() | |
| predicted_line_id = str(row.get("predicted_line_id") or "").strip() | |
| if not reference_line_id or not predicted_line_id: | |
| continue | |
| bbox = _read_bbox(str(row.get("reference_bbox") or ""), context=f"{path}:{row_number} reference") | |
| rows.append( | |
| { | |
| "reference_line_id": reference_line_id, | |
| "reference_order": _line_number(reference_line_id), | |
| "predicted_line_id": predicted_line_id, | |
| "page_index": int(row.get("page_index") or 0), | |
| "x": bbox.x, | |
| "y": bbox.y, | |
| "w": bbox.w, | |
| "h": bbox.h, | |
| "cx": bbox.x + bbox.w / 2.0, | |
| "cy": bbox.y + bbox.h / 2.0, | |
| } | |
| ) | |
| rows.sort(key=lambda item: (item["page_index"], item["reference_order"])) | |
| return rows | |
| def _is_jump(previous: dict[str, Any], current: dict[str, Any], *, min_dx: float, min_upward_dy: float, min_downward_dy: float) -> bool: | |
| dx = float(current["cx"]) - float(previous["cx"]) | |
| dy = float(current["y"]) - float(previous["y"]) | |
| return abs(dx) > min_dx or dy < -min_upward_dy or dy > min_downward_dy | |
| def _chunk_summary(chunk: list[dict[str, Any]], *, chunk_index: int) -> dict[str, Any]: | |
| return { | |
| "chunk_index": chunk_index, | |
| "start_reference_line_id": chunk[0]["reference_line_id"], | |
| "end_reference_line_id": chunk[-1]["reference_line_id"], | |
| "start_reference_order": chunk[0]["reference_order"], | |
| "end_reference_order": chunk[-1]["reference_order"], | |
| "line_count": len(chunk), | |
| "cx_median": statistics.median(float(row["cx"]) for row in chunk), | |
| "x_min": min(float(row["x"]) for row in chunk), | |
| "x_max": max(float(row["x"]) + float(row["w"]) for row in chunk), | |
| "y_min": min(float(row["y"]) for row in chunk), | |
| "y_max": max(float(row["y"]) for row in chunk), | |
| "width_median": statistics.median(float(row["w"]) for row in chunk), | |
| "predicted_line_ids": [row["predicted_line_id"] for row in chunk], | |
| } | |
| def analyze(args: argparse.Namespace) -> dict[str, Any]: | |
| alignment_path = Path(args.line_alignment) | |
| out = Path(args.out) | |
| out.mkdir(parents=True, exist_ok=True) | |
| rows = _read_matched_rows(alignment_path) | |
| chunks: list[list[dict[str, Any]]] = [] | |
| jumps: list[dict[str, Any]] = [] | |
| current: list[dict[str, Any]] = [] | |
| for row in rows: | |
| if current and _is_jump( | |
| current[-1], | |
| row, | |
| min_dx=float(args.min_dx), | |
| min_upward_dy=float(args.min_upward_dy), | |
| min_downward_dy=float(args.min_downward_dy), | |
| ): | |
| previous = current[-1] | |
| jumps.append( | |
| { | |
| "from_reference_line_id": previous["reference_line_id"], | |
| "to_reference_line_id": row["reference_line_id"], | |
| "dx": float(row["cx"]) - float(previous["cx"]), | |
| "dy": float(row["y"]) - float(previous["y"]), | |
| "from_cx": previous["cx"], | |
| "from_y": previous["y"], | |
| "to_cx": row["cx"], | |
| "to_y": row["y"], | |
| } | |
| ) | |
| chunks.append(current) | |
| current = [] | |
| current.append(row) | |
| if current: | |
| chunks.append(current) | |
| chunk_summaries = [_chunk_summary(chunk, chunk_index=index) for index, chunk in enumerate(chunks, start=1)] | |
| chunks_path = out / CHUNKS_TSV | |
| with chunks_path.open("w", encoding="utf-8", newline="") as handle: | |
| fields = [ | |
| "chunk_index", | |
| "start_reference_line_id", | |
| "end_reference_line_id", | |
| "line_count", | |
| "cx_median", | |
| "x_min", | |
| "x_max", | |
| "y_min", | |
| "y_max", | |
| "width_median", | |
| ] | |
| writer = csv.DictWriter(handle, fieldnames=fields, delimiter="\t") | |
| writer.writeheader() | |
| for chunk in chunk_summaries: | |
| writer.writerow({field: chunk[field] for field in fields}) | |
| jumps_path = out / JUMPS_TSV | |
| with jumps_path.open("w", encoding="utf-8", newline="") as handle: | |
| fields = ["from_reference_line_id", "to_reference_line_id", "dx", "dy", "from_cx", "from_y", "to_cx", "to_y"] | |
| writer = csv.DictWriter(handle, fieldnames=fields, delimiter="\t") | |
| writer.writeheader() | |
| for jump in jumps: | |
| writer.writerow(jump) | |
| report_path = out / REPORT_JSON | |
| report = { | |
| "tool": "tools/analyze_page_reading_order_geometry.py", | |
| "created_at_utc": datetime.now(UTC).isoformat(), | |
| "line_alignment": str(alignment_path), | |
| "line_alignment_sha256": sha256_file(alignment_path), | |
| "output_dir": str(out), | |
| "matched_line_count": len(rows), | |
| "chunk_count": len(chunk_summaries), | |
| "jump_count": len(jumps), | |
| "config": { | |
| "min_dx": float(args.min_dx), | |
| "min_upward_dy": float(args.min_upward_dy), | |
| "min_downward_dy": float(args.min_downward_dy), | |
| }, | |
| "chunks": chunk_summaries, | |
| "jumps": jumps, | |
| "chunks_tsv": str(chunks_path), | |
| "chunks_tsv_sha256": sha256_file(chunks_path), | |
| "jumps_tsv": str(jumps_path), | |
| "jumps_tsv_sha256": sha256_file(jumps_path), | |
| "claim_boundary": "diagnostic geometry analysis of eval alignment; use to design non-oracle reading-order policy, not as OCR output", | |
| } | |
| report_path.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") | |
| return report | |
| def main() -> int: | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument("--line-alignment", type=Path, required=True) | |
| parser.add_argument("--out", type=Path, required=True) | |
| parser.add_argument("--min-dx", type=float, default=350.0) | |
| parser.add_argument("--min-upward-dy", type=float, default=200.0) | |
| parser.add_argument("--min-downward-dy", type=float, default=120.0) | |
| args = parser.parse_args() | |
| report = analyze(args) | |
| print(str(Path(report["output_dir"]) / REPORT_JSON)) | |
| print(f"chunks={report['chunk_count']} jumps={report['jump_count']} matched={report['matched_line_count']}") | |
| return 0 | |
| if __name__ == "__main__": | |
| raise SystemExit(main()) | |