Add 1000 files (ocr-tech/tests/test_audit_limbu_book_missed_line_review_coverage.py ...)
d82db76 verified Download ocr-tech-mps-integration/src/ocrtech/references.py from prosperious/ocr-projects-archive: direct link, hf CLI and curl.
- Browser
- Download file 28.7 kB
-
https://huggingface.co/prosperious/ocr-projects-archive/resolve/main/ocr-tech-mps-integration/src/ocrtech/references.py
- Command line
-
hf download hf://prosperious/ocr-projects-archive/ocr-tech-mps-integration/src/ocrtech/references.py
-
curl -L -o references.py https://huggingface.co/prosperious/ocr-projects-archive/resolve/main/ocr-tech-mps-integration/src/ocrtech/references.py
28.7 kB
| """Reference document loading and scoring helpers.""" | |
| from __future__ import annotations | |
| import json | |
| from dataclasses import dataclass, field | |
| from pathlib import Path | |
| from typing import Any | |
| from .errors import DataValidationError | |
| from .manifest import ManifestEntry, load_manifest, write_manifest | |
| from .markdown import render_document_markdown | |
| from .metrics import bbox_iou, cer, detection_score, reading_order_pair_accuracy, table_cell_score, table_structure_similarity, table_teds_structure, table_teds_text, wer | |
| from .normalization import normalize_ocr_text | |
| from .schemas import BBox, Document, Figure, TableCell | |
| from .tables import cells_to_grid, table_to_grid | |
| class ReferenceTable: | |
| cells: list[TableCell] = field(default_factory=list) | |
| def grid(self) -> list[list[str]]: | |
| return cells_to_grid(self.cells) | |
| class ReferenceFigure: | |
| caption: str | None = None | |
| bbox: BBox | None = None | |
| page_index: int = 0 | |
| figure_id: str | None = None | |
| line_ids: list[str] = field(default_factory=list) | |
| class ReferenceDocument: | |
| text: str | None = None | |
| markdown: str | None = None | |
| reading_order: list[str] | None = None | |
| tables: list[ReferenceTable] = field(default_factory=list) | |
| figures: list[ReferenceFigure] = field(default_factory=list) | |
| metadata: dict[str, Any] = field(default_factory=dict) | |
| class ReferencePackReport: | |
| references_dir: str | |
| source_manifest: str | |
| rewritten_manifest: str | None | |
| sample_count: int | |
| summary_json_path: str | |
| summary_md_path: str | |
| def to_dict(self) -> dict[str, Any]: | |
| return { | |
| "references_dir": self.references_dir, | |
| "source_manifest": self.source_manifest, | |
| "rewritten_manifest": self.rewritten_manifest, | |
| "sample_count": self.sample_count, | |
| "summary_json_path": self.summary_json_path, | |
| "summary_md_path": self.summary_md_path, | |
| } | |
| class ReferenceAuditReport: | |
| manifest_path: str | |
| output_dir: str | |
| sample_count: int | |
| parsed_count: int | |
| text_reference_count: int | |
| markdown_reference_count: int | |
| reading_order_reference_count: int | |
| table_reference_count: int | |
| figure_reference_count: int | |
| claim_ready_reference_count: int | |
| require_claim_ready: bool | |
| issues: list[str] = field(default_factory=list) | |
| warnings: list[str] = field(default_factory=list) | |
| summary_json_path: str | None = None | |
| summary_md_path: str | None = None | |
| def passed(self) -> bool: | |
| return not self.issues | |
| def to_dict(self) -> dict[str, Any]: | |
| return { | |
| "passed": self.passed, | |
| "manifest_path": self.manifest_path, | |
| "output_dir": self.output_dir, | |
| "sample_count": self.sample_count, | |
| "parsed_count": self.parsed_count, | |
| "text_reference_count": self.text_reference_count, | |
| "markdown_reference_count": self.markdown_reference_count, | |
| "reading_order_reference_count": self.reading_order_reference_count, | |
| "table_reference_count": self.table_reference_count, | |
| "figure_reference_count": self.figure_reference_count, | |
| "claim_ready_reference_count": self.claim_ready_reference_count, | |
| "require_claim_ready": self.require_claim_ready, | |
| "issues": self.issues, | |
| "warnings": self.warnings, | |
| "summary_json_path": self.summary_json_path, | |
| "summary_md_path": self.summary_md_path, | |
| } | |
| def load_reference(input_path: Path, reference_root: Path | None = None, *, explicit_path: str | Path | None = None) -> ReferenceDocument | None: | |
| if explicit_path is not None: | |
| explicit_candidate = Path(explicit_path) | |
| if _is_json_reference_path(explicit_candidate): | |
| payload = _load_reference_json_candidates([explicit_candidate]) | |
| if payload is not None: | |
| return _reference_from_payload(payload) | |
| else: | |
| text_reference = _load_reference_text_candidates([explicit_candidate]) | |
| if text_reference is not None: | |
| return _reference_from_text_path(text_reference[0], text_reference[1]) | |
| payload = _load_reference_json(input_path, reference_root) | |
| if payload is not None: | |
| return _reference_from_payload(payload) | |
| text_reference = _load_reference_text(input_path, reference_root) | |
| if text_reference is not None: | |
| return _reference_from_text_path(text_reference[0], text_reference[1]) | |
| return None | |
| def audit_references( | |
| manifest_path: str | Path, | |
| output_dir: str | Path, | |
| *, | |
| require_claim_ready: bool = False, | |
| ) -> ReferenceAuditReport: | |
| manifest = Path(manifest_path) | |
| entries = load_manifest(manifest) | |
| out = Path(output_dir) | |
| out.mkdir(parents=True, exist_ok=True) | |
| issues: list[str] = [] | |
| warnings: list[str] = [] | |
| parsed_count = 0 | |
| text_count = 0 | |
| markdown_count = 0 | |
| reading_order_count = 0 | |
| table_count = 0 | |
| figure_count = 0 | |
| claim_ready_count = 0 | |
| for entry in entries: | |
| input_path = _resolve_manifest_path(entry.image_path, manifest.parent) | |
| explicit = entry.metadata.get("reference_path") if entry.metadata else None | |
| explicit_path = _resolve_manifest_path(explicit, manifest.parent) if isinstance(explicit, str) and explicit else None | |
| if isinstance(explicit, str) and explicit and not explicit_path.exists(): | |
| issues.append(f"{entry.sample_id}: reference_path does not exist: {explicit}") | |
| continue | |
| try: | |
| reference = load_reference(input_path, explicit_path=explicit_path) | |
| except Exception as exc: | |
| issues.append(f"{entry.sample_id}: reference failed to parse: {exc}") | |
| continue | |
| if reference is None: | |
| issues.append(f"{entry.sample_id}: no reference found") | |
| continue | |
| parsed_count += 1 | |
| if reference.text is not None: | |
| text_count += 1 | |
| if reference.markdown is not None: | |
| markdown_count += 1 | |
| if reference.reading_order: | |
| reading_order_count += 1 | |
| if reference.tables: | |
| table_count += 1 | |
| if reference.figures: | |
| figure_count += 1 | |
| metadata = reference.metadata or {} | |
| claim_ready = metadata.get("reference_status") == "verified" and metadata.get("claim_evidence_eligible") is True | |
| if claim_ready: | |
| claim_ready_count += 1 | |
| elif require_claim_ready: | |
| issues.append(f"{entry.sample_id}: reference is not claim-ready") | |
| if reference.text is None and reference.markdown is None and not reference.reading_order and not reference.tables and not reference.figures: | |
| warnings.append(f"{entry.sample_id}: reference parsed but has no scoreable fields") | |
| report = ReferenceAuditReport( | |
| manifest_path=str(manifest), | |
| output_dir=str(out), | |
| sample_count=len(entries), | |
| parsed_count=parsed_count, | |
| text_reference_count=text_count, | |
| markdown_reference_count=markdown_count, | |
| reading_order_reference_count=reading_order_count, | |
| table_reference_count=table_count, | |
| figure_reference_count=figure_count, | |
| claim_ready_reference_count=claim_ready_count, | |
| require_claim_ready=require_claim_ready, | |
| issues=issues, | |
| warnings=warnings, | |
| summary_json_path=str(out / "reference-audit.json"), | |
| summary_md_path=str(out / "reference-audit.md"), | |
| ) | |
| _write_reference_audit(report) | |
| return report | |
| def materialize_references( | |
| manifest_path: str | Path, | |
| output_dir: str | Path, | |
| *, | |
| rewritten_manifest_path: str | Path | None = None, | |
| ) -> ReferencePackReport: | |
| entries = load_manifest(manifest_path) | |
| if not entries: | |
| raise DataValidationError("reference materialization requires a non-empty manifest") | |
| out = Path(output_dir) | |
| out.mkdir(parents=True, exist_ok=True) | |
| rewritten_entries: list[ManifestEntry] = [] | |
| for entry in entries: | |
| sample_dir = out / entry.sample_id | |
| sample_dir.mkdir(parents=True, exist_ok=True) | |
| reference_path = sample_dir / "document.md" | |
| reference_path.write_text(_reference_text_from_manifest_entry(entry), encoding="utf-8") | |
| metadata = dict(entry.metadata or {}) | |
| metadata["reference_path"] = str(reference_path) | |
| rewritten_entries.append( | |
| ManifestEntry( | |
| sample_id=entry.sample_id, | |
| dataset=entry.dataset, | |
| split=entry.split, | |
| image_path=entry.image_path, | |
| text=entry.text, | |
| sha256=entry.sha256, | |
| metadata=metadata, | |
| ) | |
| ) | |
| rewritten_manifest = Path(rewritten_manifest_path) if rewritten_manifest_path is not None else None | |
| if rewritten_manifest is not None: | |
| rewritten_manifest.parent.mkdir(parents=True, exist_ok=True) | |
| write_manifest(rewritten_entries, rewritten_manifest) | |
| summary_json = out / "reference-pack.json" | |
| summary_md = out / "reference-pack.md" | |
| report = ReferencePackReport( | |
| references_dir=str(out), | |
| source_manifest=str(Path(manifest_path)), | |
| rewritten_manifest=str(rewritten_manifest) if rewritten_manifest is not None else None, | |
| sample_count=len(entries), | |
| summary_json_path=str(summary_json), | |
| summary_md_path=str(summary_md), | |
| ) | |
| summary_json.write_text(json.dumps(report.to_dict(), ensure_ascii=False, indent=2) + "\n", encoding="utf-8") | |
| summary_md.write_text( | |
| "\n".join( | |
| [ | |
| "# Reference Pack", | |
| "", | |
| f"Source manifest: `{manifest_path}`", | |
| f"References dir: `{out}`", | |
| f"Samples: `{len(entries)}`", | |
| f"Rewritten manifest: `{rewritten_manifest or ''}`", | |
| ] | |
| ) | |
| + "\n", | |
| encoding="utf-8", | |
| ) | |
| return report | |
| def _write_reference_audit(report: ReferenceAuditReport) -> None: | |
| json_path = Path(report.summary_json_path or Path(report.output_dir) / "reference-audit.json") | |
| md_path = Path(report.summary_md_path or Path(report.output_dir) / "reference-audit.md") | |
| json_path.write_text(json.dumps(report.to_dict(), ensure_ascii=False, indent=2) + "\n", encoding="utf-8") | |
| lines = [ | |
| "# Reference Audit", | |
| "", | |
| f"- passed: {report.passed}", | |
| f"- manifest: `{report.manifest_path}`", | |
| f"- samples: {report.sample_count}", | |
| f"- parsed references: {report.parsed_count}", | |
| f"- text references: {report.text_reference_count}", | |
| f"- markdown references: {report.markdown_reference_count}", | |
| f"- reading-order references: {report.reading_order_reference_count}", | |
| f"- table references: {report.table_reference_count}", | |
| f"- figure references: {report.figure_reference_count}", | |
| f"- claim-ready references: {report.claim_ready_reference_count}", | |
| f"- require claim-ready: {report.require_claim_ready}", | |
| "", | |
| "## Issues", | |
| "", | |
| ] | |
| lines.extend(f"- {issue}" for issue in report.issues) if report.issues else lines.append("- none") | |
| lines.extend(["", "## Warnings", ""]) | |
| lines.extend(f"- {warning}" for warning in report.warnings) if report.warnings else lines.append("- none") | |
| md_path.write_text("\n".join(lines) + "\n", encoding="utf-8") | |
| def score_document(document: Document | None, text: str, reference: ReferenceDocument | None) -> dict[str, float]: | |
| if reference is None: | |
| return {} | |
| metrics: dict[str, float] = {} | |
| reconstruction_text_similarity: float | None = None | |
| reconstruction_components: list[float] = [] | |
| if reference.text is not None: | |
| pred = text.strip() | |
| ref = reference.text.strip() | |
| metrics["cer"] = cer(pred, ref) | |
| metrics["wer"] = wer(pred, ref) | |
| metrics["text_cer"] = metrics["cer"] | |
| metrics["text_wer"] = metrics["wer"] | |
| if document is not None and reference.markdown is not None: | |
| predicted_markdown = render_document_markdown(document).strip() | |
| reference_markdown = reference.markdown.strip() | |
| reconstruction_cer = cer(predicted_markdown, reference_markdown) | |
| metrics["document_reconstruction_cer"] = reconstruction_cer | |
| metrics["document_reconstruction_wer"] = wer(predicted_markdown, reference_markdown) | |
| reconstruction_text_similarity = max(0.0, 1.0 - reconstruction_cer) | |
| metrics["document_reconstruction_text_similarity"] = reconstruction_text_similarity | |
| reconstruction_components.append(reconstruction_text_similarity) | |
| if document is not None and reference.reading_order: | |
| predicted_order = _document_reading_order(document) | |
| reading_order_score = reading_order_pair_accuracy(predicted_order, reference.reading_order) | |
| metrics["reading_order_pair_accuracy"] = reading_order_score | |
| if reconstruction_text_similarity is not None: | |
| reconstruction_components.append(reading_order_score) | |
| if document is not None and reference.tables: | |
| table_scores = [] | |
| structure_scores: list[float] = [] | |
| teds_structure_scores: list[float] = [] | |
| teds_text_scores: list[float] = [] | |
| exact_matches: list[float] = [] | |
| predicted_tables = sorted(document.tables, key=lambda item: (item.page_index, item.bbox.y, item.bbox.x)) | |
| table_matches = _match_tables(predicted_tables, reference.tables) | |
| for reference_table, predicted_table in table_matches: | |
| predicted_grid = table_to_grid(predicted_table) if predicted_table is not None else [] | |
| predicted_cells = predicted_table.cells if predicted_table is not None else [] | |
| table_scores.append(table_cell_score(predicted_grid, reference_table.grid)) | |
| structure_scores.append(table_structure_similarity(predicted_cells, reference_table.cells)) | |
| teds_structure_scores.append(table_teds_structure(predicted_cells, reference_table.cells)) | |
| teds_text_scores.append(table_teds_text(predicted_cells, reference_table.cells)) | |
| exact_matches.append(1.0 if _table_cells_exact(predicted_cells, reference_table.cells) else 0.0) | |
| if table_scores: | |
| metrics["table_cell_precision"] = sum(item.precision for item in table_scores) / len(table_scores) | |
| metrics["table_cell_recall"] = sum(item.recall for item in table_scores) / len(table_scores) | |
| metrics["table_cell_f1"] = sum(item.f1 for item in table_scores) / len(table_scores) | |
| metrics["table_exact_match"] = sum(exact_matches) / len(exact_matches) | |
| metrics["table_structure_similarity"] = sum(structure_scores) / len(structure_scores) | |
| metrics["table_teds_structure"] = sum(teds_structure_scores) / len(teds_structure_scores) | |
| metrics["table_teds_text"] = sum(teds_text_scores) / len(teds_text_scores) | |
| if reconstruction_text_similarity is not None: | |
| reconstruction_components.append(min(metrics["table_cell_f1"], metrics["table_teds_text"])) | |
| if document is not None and reference.figures: | |
| figure_metrics = _score_figures(document.figures, reference.figures) | |
| metrics.update(figure_metrics) | |
| if reconstruction_text_similarity is not None: | |
| if "figure_detection_f1" in figure_metrics: | |
| reconstruction_components.append(figure_metrics["figure_detection_f1"]) | |
| if "figure_caption_cer" in figure_metrics: | |
| reconstruction_components.append(max(0.0, 1.0 - figure_metrics["figure_caption_cer"])) | |
| if reconstruction_text_similarity is not None: | |
| metrics["document_reconstruction_similarity"] = min(reconstruction_components) if reconstruction_components else reconstruction_text_similarity | |
| return metrics | |
| def _match_tables(predicted_tables: list[Any], reference_tables: list[ReferenceTable]) -> list[tuple[ReferenceTable, Any | None]]: | |
| matches: list[tuple[ReferenceTable, Any | None]] = [] | |
| unused = set(range(len(predicted_tables))) | |
| for reference_table in reference_tables: | |
| best_index: int | None = None | |
| best_score: tuple[float, float, float] | None = None | |
| for index in sorted(unused): | |
| predicted_table = predicted_tables[index] | |
| score = _table_match_score(predicted_table, reference_table) | |
| if best_score is None or score > best_score: | |
| best_score = score | |
| best_index = index | |
| if best_index is None: | |
| matches.append((reference_table, None)) | |
| continue | |
| unused.remove(best_index) | |
| matches.append((reference_table, predicted_tables[best_index])) | |
| return matches | |
| def _table_match_score(predicted_table: Any, reference_table: ReferenceTable) -> tuple[float, float, float]: | |
| predicted_cells = predicted_table.cells if predicted_table is not None else [] | |
| predicted_grid = table_to_grid(predicted_table) if predicted_table is not None else [] | |
| cell_score = table_cell_score(predicted_grid, reference_table.grid) | |
| return ( | |
| table_teds_text(predicted_cells, reference_table.cells), | |
| table_teds_structure(predicted_cells, reference_table.cells), | |
| cell_score.f1, | |
| ) | |
| def _document_reading_order(document: Document) -> list[str]: | |
| ordered: list[str] = [] | |
| for page in sorted(document.pages, key=lambda item: item.page_index): | |
| for block in sorted(page.blocks, key=lambda item: item.order): | |
| if block.line_ids: | |
| ordered.extend(line_id for line_id in block.line_ids if line_id) | |
| else: | |
| ordered.append(block.block_id) | |
| return ordered | |
| def _load_reference_json(input_path: Path, reference_root: Path | None) -> dict[str, Any] | None: | |
| candidates: list[Path] = [] | |
| if reference_root: | |
| candidates.extend( | |
| [ | |
| reference_root / f"{input_path.stem}.json", | |
| reference_root / input_path.name / "reference.json", | |
| reference_root / input_path.stem / "reference.json", | |
| ] | |
| ) | |
| candidates.extend([input_path.with_suffix(input_path.suffix + ".ref.json"), input_path.with_suffix(".ref.json")]) | |
| return _load_reference_json_candidates(candidates) | |
| def _resolve_manifest_path(path_value: str | None, manifest_dir: Path) -> Path: | |
| if not path_value: | |
| return Path("") | |
| path = Path(path_value) | |
| if path.is_absolute() or path.exists(): | |
| return path | |
| candidate = manifest_dir / path | |
| if candidate.exists(): | |
| return candidate | |
| return path | |
| def _load_reference_json_candidates(candidates: list[Path]) -> dict[str, Any] | None: | |
| for candidate in candidates: | |
| if candidate.exists(): | |
| try: | |
| payload = json.loads(candidate.read_text(encoding="utf-8")) | |
| except json.JSONDecodeError as exc: | |
| raise DataValidationError(f"Invalid reference JSON {candidate}: {exc}") from exc | |
| if not isinstance(payload, dict): | |
| raise DataValidationError(f"Reference JSON must be an object: {candidate}") | |
| return payload | |
| return None | |
| def _load_reference_text(input_path: Path, reference_root: Path | None) -> tuple[str, Path] | None: | |
| candidates: list[Path] = [] | |
| if reference_root: | |
| candidates.extend( | |
| [ | |
| reference_root / f"{input_path.stem}.txt", | |
| reference_root / input_path.name / "document.md", | |
| reference_root / input_path.stem / "document.md", | |
| ] | |
| ) | |
| candidates.extend([input_path.with_suffix(input_path.suffix + ".ref.txt"), input_path.with_suffix(".ref.txt")]) | |
| return _load_reference_text_candidates(candidates) | |
| def _load_reference_text_candidates(candidates: list[Path]) -> tuple[str, Path] | None: | |
| for candidate in candidates: | |
| if candidate.exists(): | |
| return candidate.read_text(encoding="utf-8"), candidate | |
| return None | |
| def _reference_from_text_path(text: str, path: Path) -> ReferenceDocument: | |
| markdown = text if _is_markdown_reference_path(path) else None | |
| return ReferenceDocument(text=text, markdown=markdown) | |
| def _is_markdown_reference_path(path: Path) -> bool: | |
| return path.suffix.lower() in {".md", ".markdown"} or path.name == "document.md" | |
| def _reference_text_from_manifest_entry(entry: ManifestEntry) -> str: | |
| text = entry.text.rstrip("\n") | |
| return text + "\n" | |
| def _is_json_reference_path(path: Path) -> bool: | |
| return path.suffix.lower() == ".json" or path.name.endswith(".ref.json") or path.name == "reference.json" | |
| def _reference_from_payload(payload: dict[str, Any]) -> ReferenceDocument: | |
| text = payload.get("text") | |
| if text is not None and not isinstance(text, str): | |
| raise DataValidationError("reference text must be a string") | |
| markdown = _reference_markdown_from_payload(payload) | |
| reading_order = _coerce_reference_reading_order(payload.get("reading_order")) | |
| tables_payload = payload.get("tables", []) | |
| if not isinstance(tables_payload, list): | |
| raise DataValidationError("reference tables must be a list") | |
| tables = [_coerce_reference_table(item) for item in tables_payload] | |
| figures_payload = payload.get("figures", []) | |
| if not isinstance(figures_payload, list): | |
| raise DataValidationError("reference figures must be a list") | |
| figures = [_coerce_reference_figure(item) for item in figures_payload] | |
| return ReferenceDocument( | |
| text=normalize_ocr_text(text) if text is not None else None, | |
| markdown=normalize_ocr_text(markdown) if markdown is not None else None, | |
| reading_order=reading_order, | |
| tables=tables, | |
| figures=figures, | |
| metadata=dict(payload.get("metadata") or {}), | |
| ) | |
| def _coerce_reference_reading_order(value: Any) -> list[str] | None: | |
| if value is None: | |
| return None | |
| if not isinstance(value, list): | |
| raise DataValidationError("reference reading_order must be a list") | |
| ordered: list[str] = [] | |
| for index, item in enumerate(value, start=1): | |
| if isinstance(item, str): | |
| item_id = item.strip() | |
| elif isinstance(item, dict): | |
| raw_id = item.get("id") | |
| if not isinstance(raw_id, str) or not raw_id.strip(): | |
| raise DataValidationError(f"reference reading_order item {index} missing id") | |
| raw_text = item.get("text") | |
| if raw_text is not None and not isinstance(raw_text, str): | |
| raise DataValidationError(f"reference reading_order item {index} text must be a string") | |
| item_id = raw_id.strip() | |
| else: | |
| raise DataValidationError("reference reading_order must contain strings or objects with id") | |
| if not item_id: | |
| raise DataValidationError(f"reference reading_order item {index} is empty") | |
| ordered.append(item_id) | |
| return ordered | |
| def _reference_markdown_from_payload(payload: dict[str, Any]) -> str | None: | |
| for key in ("markdown", "document_markdown", "reconstruction_markdown"): | |
| value = payload.get(key) | |
| if value is None: | |
| continue | |
| if not isinstance(value, str): | |
| raise DataValidationError(f"reference {key} must be a string") | |
| return value | |
| return None | |
| def _coerce_reference_table(value: Any) -> ReferenceTable: | |
| if isinstance(value, dict): | |
| if "cells" in value and _looks_like_cell_objects(value["cells"]): | |
| return ReferenceTable(cells=[_coerce_reference_table_cell(item) for item in value["cells"]]) | |
| value = value.get("grid") or value.get("cells") | |
| if not isinstance(value, list): | |
| raise DataValidationError("reference table must be a grid or object with grid/cells") | |
| cells: list[TableCell] = [] | |
| for row_index, row in enumerate(value): | |
| if not isinstance(row, list): | |
| raise DataValidationError("reference table row must be a list") | |
| for col_index, cell in enumerate(row): | |
| cells.append(TableCell(row=row_index, col=col_index, text=normalize_ocr_text(str(cell)))) | |
| return ReferenceTable(cells=cells) | |
| def _looks_like_cell_objects(value: Any) -> bool: | |
| return isinstance(value, list) and all(isinstance(item, dict) and {"row", "col", "text"} <= set(item) for item in value) | |
| def _coerce_reference_table_cell(value: Any) -> TableCell: | |
| if not isinstance(value, dict): | |
| raise DataValidationError("reference table cell must be an object") | |
| return TableCell( | |
| row=int(value.get("row", 0)), | |
| col=int(value.get("col", 0)), | |
| text=normalize_ocr_text(str(value.get("text", ""))), | |
| rowspan=int(value.get("rowspan", 1)), | |
| colspan=int(value.get("colspan", 1)), | |
| ) | |
| def _table_cells_exact(predicted: list[TableCell], reference: list[TableCell]) -> bool: | |
| return _cell_signature(predicted) == _cell_signature(reference) | |
| def _cell_signature(cells: list[TableCell]) -> list[tuple[int, int, int, int, str]]: | |
| return sorted( | |
| ( | |
| cell.row, | |
| cell.col, | |
| cell.rowspan, | |
| cell.colspan, | |
| normalize_ocr_text(cell.text, collapse_spaces=True), | |
| ) | |
| for cell in cells | |
| ) | |
| def _coerce_reference_figure(value: Any) -> ReferenceFigure: | |
| if not isinstance(value, dict): | |
| raise DataValidationError("reference figure must be an object") | |
| bbox_payload = value.get("bbox") | |
| line_ids = value.get("line_ids", []) | |
| if not isinstance(line_ids, list) or not all(isinstance(item, str) for item in line_ids): | |
| raise DataValidationError("reference figure line_ids must be a list of strings") | |
| caption = value.get("caption") | |
| if caption is not None and not isinstance(caption, str): | |
| raise DataValidationError("reference figure caption must be a string") | |
| return ReferenceFigure( | |
| caption=normalize_ocr_text(caption) if caption is not None else None, | |
| bbox=BBox.from_any(bbox_payload) if bbox_payload is not None else None, | |
| page_index=int(value.get("page_index", 0)), | |
| figure_id=value.get("figure_id"), | |
| line_ids=line_ids, | |
| ) | |
| def _score_figures(predicted: list[Figure], reference: list[ReferenceFigure]) -> dict[str, float]: | |
| matches = _match_figures(predicted, reference) | |
| score = detection_score(len(predicted), len(reference), len(matches)) | |
| metrics: dict[str, float] = { | |
| "figure_detection_precision": score.precision, | |
| "figure_detection_recall": score.recall, | |
| "figure_detection_f1": score.f1, | |
| } | |
| caption_errors: list[float] = [] | |
| for predicted_index, reference_index in matches: | |
| ref_caption = reference[reference_index].caption | |
| pred_caption = predicted[predicted_index].caption or predicted[predicted_index].summary | |
| if ref_caption is not None: | |
| caption_errors.append(cer(pred_caption or "", ref_caption)) | |
| if caption_errors: | |
| metrics["figure_caption_cer"] = sum(caption_errors) / len(caption_errors) | |
| return metrics | |
| def _match_figures(predicted: list[Figure], reference: list[ReferenceFigure]) -> list[tuple[int, int]]: | |
| used_predicted: set[int] = set() | |
| matches: list[tuple[int, int]] = [] | |
| for ref_index, ref in enumerate(reference): | |
| best: tuple[float, int] | None = None | |
| for pred_index, pred in enumerate(predicted): | |
| if pred_index in used_predicted or pred.page_index != ref.page_index: | |
| continue | |
| score = _figure_match_score(pred, ref) | |
| if score <= 0: | |
| continue | |
| if best is None or score > best[0]: | |
| best = (score, pred_index) | |
| if best is not None: | |
| used_predicted.add(best[1]) | |
| matches.append((best[1], ref_index)) | |
| return matches | |
| def _figure_match_score(predicted: Figure, reference: ReferenceFigure) -> float: | |
| if reference.bbox is not None: | |
| iou = bbox_iou(predicted.bbox, reference.bbox) | |
| if iou >= 0.5: | |
| return iou | |
| if reference.caption and predicted.caption: | |
| caption_error = cer(predicted.caption, reference.caption) | |
| if caption_error <= 0.25: | |
| return 1.0 - caption_error | |
| if reference.line_ids: | |
| source_ids = set(predicted.metadata.get("line_ids", [])) if isinstance(predicted.metadata, dict) else set() | |
| if source_ids & set(reference.line_ids): | |
| return 0.75 | |
| return 0.0 | |