ocr-projects-archive / ocr-tech /tools /apply_page_reading_order_geometry_profile.py
prosperious's picture
Add 1000 files (ocr-tech/tests/test_audit_limbu_book_missed_line_review_coverage.py ...)
d82db76 verified
Raw History Blame Contribute Delete
18.3 kB
#!/usr/bin/env python3
"""Apply a reading-order geometry profile to page OCR document projections."""
from __future__ import annotations
import argparse
import json
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
from ocrtech.markdown import render_document_markdown
from ocrtech.manifest import sha256_file
from ocrtech.schemas import Block, Document, TextLine
REPORT_JSON = "reading-order-geometry-profile-application.json"
ASSIGNMENTS_JSONL = "reading-order-geometry-profile-assignments.jsonl"
def _read_json_object(path: Path) -> dict[str, Any]:
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except FileNotFoundError as exc:
raise ValueError(f"required JSON file is missing: {path}") from exc
except json.JSONDecodeError as exc:
raise ValueError(f"invalid JSON in {path}: {exc}") from exc
if not isinstance(payload, dict):
raise ValueError(f"JSON file must contain an object: {path}")
return payload
def _document_paths(documents_dir: Path) -> dict[str, Path]:
if not documents_dir.is_dir():
raise ValueError(f"documents directory does not exist: {documents_dir}")
paths: dict[str, Path] = {}
for child in sorted(documents_dir.iterdir()):
if not child.is_dir():
continue
document_path = child / "document.json"
if document_path.is_file():
paths[child.name] = document_path
if not paths:
raise ValueError(f"documents directory contains no sample document.json files: {documents_dir}")
return paths
def _read_chunks_from_profile(profile_path: Path, profile: dict[str, Any]) -> list[dict[str, Any]]:
chunks = profile.get("chunks")
if not isinstance(chunks, list) or not chunks:
raise ValueError(f"reading-order profile must contain non-empty chunks list: {profile_path}")
normalized: list[dict[str, Any]] = []
for index, chunk in enumerate(chunks, start=1):
if not isinstance(chunk, dict):
raise ValueError(f"profile chunk {index} must be an object")
try:
chunk_index = int(chunk.get("chunk_index") or index)
cx_median = float(chunk["cx_median"])
y_min = float(chunk["y_min"])
y_max = float(chunk["y_max"])
width_median = float(chunk["width_median"])
except (KeyError, TypeError, ValueError) as exc:
raise ValueError(f"profile chunk {index} missing numeric chunk_index/cx_median/y_min/y_max/width_median") from exc
if width_median <= 0:
raise ValueError(f"profile chunk {index} width_median must be positive")
normalized.append(
{
"chunk_index": chunk_index,
"cx_median": cx_median,
"y_min": y_min,
"y_max": y_max,
"width_median": width_median,
"start_line_id": str(chunk.get("start_line_id") or ""),
"end_line_id": str(chunk.get("end_line_id") or ""),
"start_reference_line_id": str(chunk.get("start_reference_line_id") or ""),
"end_reference_line_id": str(chunk.get("end_reference_line_id") or ""),
}
)
normalized.sort(key=lambda item: item["chunk_index"])
return normalized
def _resolve_profile_path(value: str, base_dir: Path) -> Path:
path = Path(value)
if path.is_absolute() or path.exists():
return path
candidate = base_dir / path
if candidate.exists():
return candidate
return path
def _read_profile_map(profile_map_path: Path) -> tuple[dict[str, Path], Path | None]:
payload = _read_json_object(profile_map_path)
base_dir = profile_map_path.parent
sample_profiles: dict[str, Path] = {}
default_profile: Path | None = None
if "profiles" in payload:
profiles = payload.get("profiles")
if not isinstance(profiles, list):
raise ValueError(f"profile map profiles must be a list: {profile_map_path}")
for index, row in enumerate(profiles, start=1):
if not isinstance(row, dict):
raise ValueError(f"profile map row {index} must be an object")
sample_id = str(row.get("sample_id") or "").strip()
profile_value = row.get("profile")
if not sample_id:
raise ValueError(f"profile map row {index} missing sample_id")
if not isinstance(profile_value, str) or not profile_value:
raise ValueError(f"profile map row {index} missing profile")
if sample_id in sample_profiles:
raise ValueError(f"profile map has duplicate sample_id: {sample_id}")
sample_profiles[sample_id] = _resolve_profile_path(profile_value, base_dir)
else:
for sample_id, profile_value in payload.items():
if sample_id == "default_profile":
continue
if not isinstance(profile_value, str) or not profile_value:
raise ValueError(f"profile map value for {sample_id!r} must be a profile path string")
sample_profiles[str(sample_id)] = _resolve_profile_path(profile_value, base_dir)
default_value = payload.get("default_profile")
if isinstance(default_value, str) and default_value:
default_profile = _resolve_profile_path(default_value, base_dir)
return sample_profiles, default_profile
def _load_profile_bundle(
profile_path: Path,
*,
require_profile_claim_safe: bool,
cache: dict[Path, dict[str, Any]],
) -> dict[str, Any]:
resolved = profile_path.resolve()
if resolved in cache:
return cache[resolved]
profile = _read_json_object(profile_path)
if require_profile_claim_safe and profile.get("profile_claim_safe") is not True:
raise ValueError(f"--require-profile-claim-safe requested but profile_claim_safe is not true: {profile_path}")
bundle = {
"path": profile_path,
"sha256": sha256_file(profile_path),
"profile_claim_safe": bool(profile.get("profile_claim_safe", False)),
"chunks": _read_chunks_from_profile(profile_path, profile),
}
cache[resolved] = bundle
return bundle
def _profile_bundle_for_sample(
sample_id: str,
*,
single_profile_path: Path | None,
profile_map: dict[str, Path],
default_profile: Path | None,
require_profile_claim_safe: bool,
cache: dict[Path, dict[str, Any]],
) -> dict[str, Any]:
profile_path = single_profile_path or profile_map.get(sample_id) or default_profile
if profile_path is None:
raise ValueError(f"no reading-order profile configured for sample_id={sample_id!r}")
return _load_profile_bundle(
profile_path,
require_profile_claim_safe=require_profile_claim_safe,
cache=cache,
)
def _chunk_score(
line: TextLine,
chunk: dict[str, Any],
*,
x_scale: float,
y_scale: float,
outside_weight: float,
) -> float:
center_x = line.bbox.x + line.bbox.w / 2.0
x_scale_value = max(80.0, float(chunk["width_median"]) * x_scale)
dx = abs(center_x - float(chunk["cx_median"])) / x_scale_value
if line.bbox.y < float(chunk["y_min"]):
outside_y = float(chunk["y_min"]) - line.bbox.y
elif line.bbox.y > float(chunk["y_max"]):
outside_y = line.bbox.y - float(chunk["y_max"])
else:
outside_y = 0.0
dy = outside_y / y_scale
return dx + outside_weight * dy
def _assign_line(
line: TextLine,
chunks: list[dict[str, Any]],
*,
x_scale: float,
y_scale: float,
outside_weight: float,
) -> tuple[dict[str, Any], float]:
best_chunk = chunks[0]
best_score = _chunk_score(line, best_chunk, x_scale=x_scale, y_scale=y_scale, outside_weight=outside_weight)
for chunk in chunks[1:]:
score = _chunk_score(line, chunk, x_scale=x_scale, y_scale=y_scale, outside_weight=outside_weight)
if score < best_score:
best_chunk = chunk
best_score = score
return best_chunk, best_score
def _block_key(block: Block, line_keys: dict[str, tuple[int, float, float, float, float, str]]) -> tuple[int, float, float, float, float, str]:
if block.line_ids:
key = line_keys.get(block.line_ids[0])
if key is not None:
return key
return (10**9, block.bbox.y, block.bbox.x, block.bbox.h, block.bbox.w, block.block_id)
def _write_document_artifacts(document_dir: Path, document: Document) -> None:
document_dir.mkdir(parents=True, exist_ok=True)
(document_dir / "document.json").write_text(json.dumps(document.to_dict(), ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
(document_dir / "document.md").write_text(render_document_markdown(document), encoding="utf-8")
(document_dir / "document.body.md").write_text(render_document_markdown(document, include_structural_roles=False), encoding="utf-8")
def apply_profile(args: argparse.Namespace) -> dict[str, Any]:
source_documents_dir = Path(args.source_documents_dir)
raw_profile = getattr(args, "profile", None)
raw_profile_map = getattr(args, "profile_map", None)
profile_path = Path(raw_profile) if raw_profile else None
profile_map_path = Path(raw_profile_map) if raw_profile_map else None
out = Path(args.out)
if source_documents_dir.resolve() == out.resolve():
raise ValueError("--out must not be the same directory as --source-documents-dir")
if float(args.x_scale) <= 0 or float(args.y_scale) <= 0 or float(args.outside_weight) < 0:
raise ValueError("--x-scale/--y-scale must be positive and --outside-weight must be non-negative")
if (profile_path is None) == (profile_map_path is None):
raise ValueError("exactly one of --profile or --profile-map is required")
profile_map: dict[str, Path] = {}
default_profile: Path | None = None
if profile_map_path is not None:
profile_map, default_profile = _read_profile_map(profile_map_path)
source_paths = _document_paths(source_documents_dir)
out.mkdir(parents=True, exist_ok=True)
assignments: list[dict[str, Any]] = []
page_reports: list[dict[str, Any]] = []
profile_cache: dict[Path, dict[str, Any]] = {}
profile_usage: dict[str, dict[str, Any]] = {}
require_profile_claim_safe = bool(getattr(args, "require_profile_claim_safe", False))
for sample_id, document_path in sorted(source_paths.items()):
profile_bundle = _profile_bundle_for_sample(
sample_id,
single_profile_path=profile_path,
profile_map=profile_map,
default_profile=default_profile,
require_profile_claim_safe=require_profile_claim_safe,
cache=profile_cache,
)
chunks = profile_bundle["chunks"]
profile_key = str(profile_bundle["path"])
profile_usage.setdefault(
profile_key,
{
"profile": profile_key,
"profile_sha256": profile_bundle["sha256"],
"profile_claim_safe": profile_bundle["profile_claim_safe"],
"chunk_count": len(chunks),
"sample_ids": [],
},
)["sample_ids"].append(sample_id)
document = Document.from_dict(_read_json_object(document_path))
for page in document.pages:
line_keys: dict[str, tuple[int, float, float, float, float, str]] = {}
for line in page.text_lines:
chunk, score = _assign_line(
line,
chunks,
x_scale=float(args.x_scale),
y_scale=float(args.y_scale),
outside_weight=float(args.outside_weight),
)
line_id = line.line_id or ""
line_keys[line_id] = (
int(chunk["chunk_index"]),
line.bbox.y,
line.bbox.x,
line.bbox.h,
line.bbox.w,
line_id,
)
metadata = line.metadata if isinstance(line.metadata, dict) else {}
metadata["reading_order_geometry_profile_assignment"] = {
"profile": str(profile_bundle["path"]),
"profile_sha256": profile_bundle["sha256"],
"chunk_index": int(chunk["chunk_index"]),
"score": score,
"start_reference_line_id": chunk.get("start_reference_line_id") or "",
"end_reference_line_id": chunk.get("end_reference_line_id") or "",
"start_line_id": chunk.get("start_line_id") or "",
"end_line_id": chunk.get("end_line_id") or "",
"claim_boundary": "profile-based reading-order assignment; profile provenance controls claim safety",
}
line.metadata = metadata
assignments.append(
{
"sample_id": sample_id,
"page_index": page.page_index,
"line_id": line_id,
"profile": str(profile_bundle["path"]),
"profile_sha256": profile_bundle["sha256"],
"chunk_index": int(chunk["chunk_index"]),
"score": score,
"bbox": line.bbox.to_list(),
"text_length": len(line.text or ""),
}
)
page.text_lines = sorted(page.text_lines, key=lambda line: line_keys.get(line.line_id or "", (10**9, line.bbox.y, line.bbox.x, line.bbox.h, line.bbox.w, line.line_id or "")))
lines_by_id = {line.line_id or "": line for line in page.text_lines}
for block in page.blocks:
if not block.line_ids:
continue
block.line_ids = sorted(
block.line_ids,
key=lambda line_id: line_keys.get(line_id, (10**9, block.bbox.y, block.bbox.x, block.bbox.h, block.bbox.w, line_id)),
)
block_lines = [lines_by_id[line_id] for line_id in block.line_ids if line_id in lines_by_id]
if block_lines:
block.text = "\n".join(line.text for line in block_lines)
page.blocks = sorted(page.blocks, key=lambda block: _block_key(block, line_keys))
for order, block in enumerate(page.blocks, start=1):
block.order = order
target_dir = out / sample_id
_write_document_artifacts(target_dir, document)
line_count = sum(len(page.text_lines) for page in document.pages)
page_reports.append(
{
"sample_id": sample_id,
"source_document": str(document_path),
"source_document_sha256": sha256_file(document_path),
"profile": str(profile_bundle["path"]),
"profile_sha256": profile_bundle["sha256"],
"profile_claim_safe": profile_bundle["profile_claim_safe"],
"document_json": str(target_dir / "document.json"),
"document_json_sha256": sha256_file(target_dir / "document.json"),
"document_md_sha256": sha256_file(target_dir / "document.md"),
"document_body_md_sha256": sha256_file(target_dir / "document.body.md"),
"line_count": line_count,
}
)
assignments_path = out / ASSIGNMENTS_JSONL
assignments_path.write_text("".join(json.dumps(row, ensure_ascii=False, sort_keys=True) + "\n" for row in assignments), encoding="utf-8")
report_path = out / REPORT_JSON
report = {
"tool": "tools/apply_page_reading_order_geometry_profile.py",
"created_at_utc": datetime.now(UTC).isoformat(),
"source_documents_dir": str(source_documents_dir),
"profile": str(profile_path) if profile_path is not None else "",
"profile_map": str(profile_map_path) if profile_map_path is not None else "",
"profile_map_sha256": sha256_file(profile_map_path) if profile_map_path is not None else "",
"profile_sha256": sha256_file(profile_path) if profile_path is not None else "",
"profile_claim_safe": all(bool(item["profile_claim_safe"]) for item in profile_usage.values()),
"profile_usage": list(profile_usage.values()),
"out": str(out),
"config": {
"x_scale": float(args.x_scale),
"y_scale": float(args.y_scale),
"outside_weight": float(args.outside_weight),
"require_profile_claim_safe": require_profile_claim_safe,
},
"chunk_count": sum(int(item["chunk_count"]) for item in profile_usage.values()),
"document_count": len(source_paths),
"assignment_count": len(assignments),
"pages": page_reports,
"assignments_jsonl": str(assignments_path),
"assignments_jsonl_sha256": sha256_file(assignments_path),
"claim_boundary": "profile-based reading-order application; not claim-safe unless profile provenance is claim-safe",
}
report["report_json"] = str(report_path)
report_path.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
return report
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--source-documents-dir", type=Path, required=True)
profile_group = parser.add_mutually_exclusive_group(required=True)
profile_group.add_argument("--profile", type=Path)
profile_group.add_argument("--profile-map", type=Path)
parser.add_argument("--out", type=Path, required=True)
parser.add_argument("--x-scale", type=float, default=1.0)
parser.add_argument("--y-scale", type=float, default=80.0)
parser.add_argument("--outside-weight", type=float, default=2.0)
parser.add_argument("--require-profile-claim-safe", action="store_true")
args = parser.parse_args()
report = apply_profile(args)
print(report["report_json"])
print(f"documents={report['document_count']} assignments={report['assignment_count']} chunks={report['chunk_count']}")
return 0
if __name__ == "__main__":
raise SystemExit(main())