ocr-tech-k2 / src /ocrtech /references.py
prosperious's picture
Add 195 files (runs/hf-recognizer-mixed-v2-full/finalized/package/artifacts/model/model.safetensors ...)
1ef1de7 verified
Raw History Blame Contribute Delete
20.1 kB
"""Reference document loading and scoring helpers."""
from __future__ import annotations
import json
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
from .errors import DataValidationError
from .manifest import ManifestEntry, load_manifest, write_manifest
from .markdown import render_document_markdown
from .metrics import bbox_iou, cer, detection_score, reading_order_pair_accuracy, table_cell_score, table_structure_similarity, table_teds_structure, table_teds_text, wer
from .normalization import normalize_ocr_text
from .schemas import BBox, Document, Figure, TableCell
from .tables import cells_to_grid, table_to_grid
@dataclass(slots=True)
class ReferenceTable:
cells: list[TableCell] = field(default_factory=list)
@property
def grid(self) -> list[list[str]]:
return cells_to_grid(self.cells)
@dataclass(slots=True)
class ReferenceFigure:
caption: str | None = None
bbox: BBox | None = None
page_index: int = 0
figure_id: str | None = None
line_ids: list[str] = field(default_factory=list)
@dataclass(slots=True)
class ReferenceDocument:
text: str | None = None
markdown: str | None = None
reading_order: list[str] | None = None
tables: list[ReferenceTable] = field(default_factory=list)
figures: list[ReferenceFigure] = field(default_factory=list)
metadata: dict[str, Any] = field(default_factory=dict)
@dataclass(slots=True)
class ReferencePackReport:
references_dir: str
source_manifest: str
rewritten_manifest: str | None
sample_count: int
summary_json_path: str
summary_md_path: str
def to_dict(self) -> dict[str, Any]:
return {
"references_dir": self.references_dir,
"source_manifest": self.source_manifest,
"rewritten_manifest": self.rewritten_manifest,
"sample_count": self.sample_count,
"summary_json_path": self.summary_json_path,
"summary_md_path": self.summary_md_path,
}
def load_reference(input_path: Path, reference_root: Path | None = None, *, explicit_path: str | Path | None = None) -> ReferenceDocument | None:
if explicit_path is not None:
explicit_candidate = Path(explicit_path)
if _is_json_reference_path(explicit_candidate):
payload = _load_reference_json_candidates([explicit_candidate])
if payload is not None:
return _reference_from_payload(payload)
else:
text_reference = _load_reference_text_candidates([explicit_candidate])
if text_reference is not None:
return _reference_from_text_path(text_reference[0], text_reference[1])
payload = _load_reference_json(input_path, reference_root)
if payload is not None:
return _reference_from_payload(payload)
text_reference = _load_reference_text(input_path, reference_root)
if text_reference is not None:
return _reference_from_text_path(text_reference[0], text_reference[1])
return None
def materialize_references(
manifest_path: str | Path,
output_dir: str | Path,
*,
rewritten_manifest_path: str | Path | None = None,
) -> ReferencePackReport:
entries = load_manifest(manifest_path)
if not entries:
raise DataValidationError("reference materialization requires a non-empty manifest")
out = Path(output_dir)
out.mkdir(parents=True, exist_ok=True)
rewritten_entries: list[ManifestEntry] = []
for entry in entries:
sample_dir = out / entry.sample_id
sample_dir.mkdir(parents=True, exist_ok=True)
reference_path = sample_dir / "document.md"
reference_path.write_text(_reference_text_from_manifest_entry(entry), encoding="utf-8")
metadata = dict(entry.metadata or {})
metadata["reference_path"] = str(reference_path)
rewritten_entries.append(
ManifestEntry(
sample_id=entry.sample_id,
dataset=entry.dataset,
split=entry.split,
image_path=entry.image_path,
text=entry.text,
sha256=entry.sha256,
metadata=metadata,
)
)
rewritten_manifest = Path(rewritten_manifest_path) if rewritten_manifest_path is not None else None
if rewritten_manifest is not None:
rewritten_manifest.parent.mkdir(parents=True, exist_ok=True)
write_manifest(rewritten_entries, rewritten_manifest)
summary_json = out / "reference-pack.json"
summary_md = out / "reference-pack.md"
report = ReferencePackReport(
references_dir=str(out),
source_manifest=str(Path(manifest_path)),
rewritten_manifest=str(rewritten_manifest) if rewritten_manifest is not None else None,
sample_count=len(entries),
summary_json_path=str(summary_json),
summary_md_path=str(summary_md),
)
summary_json.write_text(json.dumps(report.to_dict(), ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
summary_md.write_text(
"\n".join(
[
"# Reference Pack",
"",
f"Source manifest: `{manifest_path}`",
f"References dir: `{out}`",
f"Samples: `{len(entries)}`",
f"Rewritten manifest: `{rewritten_manifest or ''}`",
]
)
+ "\n",
encoding="utf-8",
)
return report
def score_document(document: Document | None, text: str, reference: ReferenceDocument | None) -> dict[str, float]:
if reference is None:
return {}
metrics: dict[str, float] = {}
if reference.text is not None:
pred = text.strip()
ref = reference.text.strip()
metrics["cer"] = cer(pred, ref)
metrics["wer"] = wer(pred, ref)
metrics["text_cer"] = metrics["cer"]
metrics["text_wer"] = metrics["wer"]
if document is not None and reference.markdown is not None:
predicted_markdown = render_document_markdown(document).strip()
reference_markdown = reference.markdown.strip()
reconstruction_cer = cer(predicted_markdown, reference_markdown)
metrics["document_reconstruction_cer"] = reconstruction_cer
metrics["document_reconstruction_wer"] = wer(predicted_markdown, reference_markdown)
metrics["document_reconstruction_similarity"] = max(0.0, 1.0 - reconstruction_cer)
if document is not None and reference.reading_order:
predicted_order = _document_reading_order(document)
metrics["reading_order_pair_accuracy"] = reading_order_pair_accuracy(predicted_order, reference.reading_order)
if document is not None and reference.tables:
table_scores = []
structure_scores: list[float] = []
teds_structure_scores: list[float] = []
teds_text_scores: list[float] = []
exact_matches: list[float] = []
predicted_tables = sorted(document.tables, key=lambda item: (item.page_index, item.bbox.y, item.bbox.x))
table_matches = _match_tables(predicted_tables, reference.tables)
for reference_table, predicted_table in table_matches:
predicted_grid = table_to_grid(predicted_table) if predicted_table is not None else []
predicted_cells = predicted_table.cells if predicted_table is not None else []
table_scores.append(table_cell_score(predicted_grid, reference_table.grid))
structure_scores.append(table_structure_similarity(predicted_cells, reference_table.cells))
teds_structure_scores.append(table_teds_structure(predicted_cells, reference_table.cells))
teds_text_scores.append(table_teds_text(predicted_cells, reference_table.cells))
exact_matches.append(1.0 if _table_cells_exact(predicted_cells, reference_table.cells) else 0.0)
if table_scores:
metrics["table_cell_precision"] = sum(item.precision for item in table_scores) / len(table_scores)
metrics["table_cell_recall"] = sum(item.recall for item in table_scores) / len(table_scores)
metrics["table_cell_f1"] = sum(item.f1 for item in table_scores) / len(table_scores)
metrics["table_exact_match"] = sum(exact_matches) / len(exact_matches)
metrics["table_structure_similarity"] = sum(structure_scores) / len(structure_scores)
metrics["table_teds_structure"] = sum(teds_structure_scores) / len(teds_structure_scores)
metrics["table_teds_text"] = sum(teds_text_scores) / len(teds_text_scores)
if document is not None and reference.figures:
figure_metrics = _score_figures(document.figures, reference.figures)
metrics.update(figure_metrics)
return metrics
def _match_tables(predicted_tables: list[Any], reference_tables: list[ReferenceTable]) -> list[tuple[ReferenceTable, Any | None]]:
matches: list[tuple[ReferenceTable, Any | None]] = []
unused = set(range(len(predicted_tables)))
for reference_table in reference_tables:
best_index: int | None = None
best_score: tuple[float, float, float] | None = None
for index in sorted(unused):
predicted_table = predicted_tables[index]
score = _table_match_score(predicted_table, reference_table)
if best_score is None or score > best_score:
best_score = score
best_index = index
if best_index is None:
matches.append((reference_table, None))
continue
unused.remove(best_index)
matches.append((reference_table, predicted_tables[best_index]))
return matches
def _table_match_score(predicted_table: Any, reference_table: ReferenceTable) -> tuple[float, float, float]:
predicted_cells = predicted_table.cells if predicted_table is not None else []
predicted_grid = table_to_grid(predicted_table) if predicted_table is not None else []
cell_score = table_cell_score(predicted_grid, reference_table.grid)
return (
table_teds_text(predicted_cells, reference_table.cells),
table_teds_structure(predicted_cells, reference_table.cells),
cell_score.f1,
)
def _document_reading_order(document: Document) -> list[str]:
ordered: list[str] = []
for page in sorted(document.pages, key=lambda item: item.page_index):
for block in sorted(page.blocks, key=lambda item: item.order):
if block.line_ids:
ordered.extend(line_id for line_id in block.line_ids if line_id)
else:
ordered.append(block.block_id)
return ordered
def _load_reference_json(input_path: Path, reference_root: Path | None) -> dict[str, Any] | None:
candidates: list[Path] = []
if reference_root:
candidates.extend(
[
reference_root / f"{input_path.stem}.json",
reference_root / input_path.name / "reference.json",
reference_root / input_path.stem / "reference.json",
]
)
candidates.extend([input_path.with_suffix(input_path.suffix + ".ref.json"), input_path.with_suffix(".ref.json")])
return _load_reference_json_candidates(candidates)
def _load_reference_json_candidates(candidates: list[Path]) -> dict[str, Any] | None:
for candidate in candidates:
if candidate.exists():
try:
payload = json.loads(candidate.read_text(encoding="utf-8"))
except json.JSONDecodeError as exc:
raise DataValidationError(f"Invalid reference JSON {candidate}: {exc}") from exc
if not isinstance(payload, dict):
raise DataValidationError(f"Reference JSON must be an object: {candidate}")
return payload
return None
def _load_reference_text(input_path: Path, reference_root: Path | None) -> tuple[str, Path] | None:
candidates: list[Path] = []
if reference_root:
candidates.extend(
[
reference_root / f"{input_path.stem}.txt",
reference_root / input_path.name / "document.md",
reference_root / input_path.stem / "document.md",
]
)
candidates.extend([input_path.with_suffix(input_path.suffix + ".ref.txt"), input_path.with_suffix(".ref.txt")])
return _load_reference_text_candidates(candidates)
def _load_reference_text_candidates(candidates: list[Path]) -> tuple[str, Path] | None:
for candidate in candidates:
if candidate.exists():
return candidate.read_text(encoding="utf-8"), candidate
return None
def _reference_from_text_path(text: str, path: Path) -> ReferenceDocument:
markdown = text if _is_markdown_reference_path(path) else None
return ReferenceDocument(text=text, markdown=markdown)
def _is_markdown_reference_path(path: Path) -> bool:
return path.suffix.lower() in {".md", ".markdown"} or path.name == "document.md"
def _reference_text_from_manifest_entry(entry: ManifestEntry) -> str:
text = entry.text.rstrip("\n")
return text + "\n"
def _is_json_reference_path(path: Path) -> bool:
return path.suffix.lower() == ".json" or path.name.endswith(".ref.json") or path.name == "reference.json"
def _reference_from_payload(payload: dict[str, Any]) -> ReferenceDocument:
text = payload.get("text")
if text is not None and not isinstance(text, str):
raise DataValidationError("reference text must be a string")
markdown = _reference_markdown_from_payload(payload)
reading_order = payload.get("reading_order")
if reading_order is not None:
if not isinstance(reading_order, list) or not all(isinstance(item, str) for item in reading_order):
raise DataValidationError("reference reading_order must be a list of strings")
tables_payload = payload.get("tables", [])
if not isinstance(tables_payload, list):
raise DataValidationError("reference tables must be a list")
tables = [_coerce_reference_table(item) for item in tables_payload]
figures_payload = payload.get("figures", [])
if not isinstance(figures_payload, list):
raise DataValidationError("reference figures must be a list")
figures = [_coerce_reference_figure(item) for item in figures_payload]
return ReferenceDocument(
text=normalize_ocr_text(text) if text is not None else None,
markdown=normalize_ocr_text(markdown) if markdown is not None else None,
reading_order=reading_order,
tables=tables,
figures=figures,
metadata=dict(payload.get("metadata") or {}),
)
def _reference_markdown_from_payload(payload: dict[str, Any]) -> str | None:
for key in ("markdown", "document_markdown", "reconstruction_markdown"):
value = payload.get(key)
if value is None:
continue
if not isinstance(value, str):
raise DataValidationError(f"reference {key} must be a string")
return value
return None
def _coerce_reference_table(value: Any) -> ReferenceTable:
if isinstance(value, dict):
if "cells" in value and _looks_like_cell_objects(value["cells"]):
return ReferenceTable(cells=[_coerce_reference_table_cell(item) for item in value["cells"]])
value = value.get("grid") or value.get("cells")
if not isinstance(value, list):
raise DataValidationError("reference table must be a grid or object with grid/cells")
cells: list[TableCell] = []
for row_index, row in enumerate(value):
if not isinstance(row, list):
raise DataValidationError("reference table row must be a list")
for col_index, cell in enumerate(row):
cells.append(TableCell(row=row_index, col=col_index, text=normalize_ocr_text(str(cell))))
return ReferenceTable(cells=cells)
def _looks_like_cell_objects(value: Any) -> bool:
return isinstance(value, list) and all(isinstance(item, dict) and {"row", "col", "text"} <= set(item) for item in value)
def _coerce_reference_table_cell(value: Any) -> TableCell:
if not isinstance(value, dict):
raise DataValidationError("reference table cell must be an object")
return TableCell(
row=int(value.get("row", 0)),
col=int(value.get("col", 0)),
text=normalize_ocr_text(str(value.get("text", ""))),
rowspan=int(value.get("rowspan", 1)),
colspan=int(value.get("colspan", 1)),
)
def _table_cells_exact(predicted: list[TableCell], reference: list[TableCell]) -> bool:
return _cell_signature(predicted) == _cell_signature(reference)
def _cell_signature(cells: list[TableCell]) -> list[tuple[int, int, int, int, str]]:
return sorted(
(
cell.row,
cell.col,
cell.rowspan,
cell.colspan,
normalize_ocr_text(cell.text, collapse_spaces=True),
)
for cell in cells
)
def _coerce_reference_figure(value: Any) -> ReferenceFigure:
if not isinstance(value, dict):
raise DataValidationError("reference figure must be an object")
bbox_payload = value.get("bbox")
line_ids = value.get("line_ids", [])
if not isinstance(line_ids, list) or not all(isinstance(item, str) for item in line_ids):
raise DataValidationError("reference figure line_ids must be a list of strings")
caption = value.get("caption")
if caption is not None and not isinstance(caption, str):
raise DataValidationError("reference figure caption must be a string")
return ReferenceFigure(
caption=normalize_ocr_text(caption) if caption is not None else None,
bbox=BBox.from_any(bbox_payload) if bbox_payload is not None else None,
page_index=int(value.get("page_index", 0)),
figure_id=value.get("figure_id"),
line_ids=line_ids,
)
def _score_figures(predicted: list[Figure], reference: list[ReferenceFigure]) -> dict[str, float]:
matches = _match_figures(predicted, reference)
score = detection_score(len(predicted), len(reference), len(matches))
metrics: dict[str, float] = {
"figure_detection_precision": score.precision,
"figure_detection_recall": score.recall,
"figure_detection_f1": score.f1,
}
caption_errors: list[float] = []
for predicted_index, reference_index in matches:
ref_caption = reference[reference_index].caption
pred_caption = predicted[predicted_index].caption or predicted[predicted_index].summary
if ref_caption is not None:
caption_errors.append(cer(pred_caption or "", ref_caption))
if caption_errors:
metrics["figure_caption_cer"] = sum(caption_errors) / len(caption_errors)
return metrics
def _match_figures(predicted: list[Figure], reference: list[ReferenceFigure]) -> list[tuple[int, int]]:
used_predicted: set[int] = set()
matches: list[tuple[int, int]] = []
for ref_index, ref in enumerate(reference):
best: tuple[float, int] | None = None
for pred_index, pred in enumerate(predicted):
if pred_index in used_predicted or pred.page_index != ref.page_index:
continue
score = _figure_match_score(pred, ref)
if score <= 0:
continue
if best is None or score > best[0]:
best = (score, pred_index)
if best is not None:
used_predicted.add(best[1])
matches.append((best[1], ref_index))
return matches
def _figure_match_score(predicted: Figure, reference: ReferenceFigure) -> float:
if reference.bbox is not None:
iou = bbox_iou(predicted.bbox, reference.bbox)
if iou >= 0.5:
return iou
if reference.caption and predicted.caption:
caption_error = cer(predicted.caption, reference.caption)
if caption_error <= 0.25:
return 1.0 - caption_error
if reference.line_ids:
source_ids = set(predicted.metadata.get("line_ids", [])) if isinstance(predicted.metadata, dict) else set()
if source_ids & set(reference.line_ids):
return 0.75
return 0.0