ocr-projects-archive / ocr-tech /tools /analyze_page_line_failures.py
prosperious's picture
Add 1000 files (ocr-tech/tests/test_audit_limbu_book_missed_line_review_coverage.py ...)
d82db76 verified
Raw History Blame Contribute Delete
43.5 kB
#!/usr/bin/env python3
"""Analyze page OCR line-alignment failures with crop provenance.
The page scorer already emits `line-alignment.csv`. This diagnostic joins those
alignment rows to an image-line crop manifest when available, then writes ranked
failure examples and aggregate buckets that distinguish geometry failures from
recognizer/crop-quality failures on well-aligned boxes.
"""
from __future__ import annotations
import argparse
import csv
import json
import platform
import ast
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
from ocrtech.manifest import sha256_file
def _read_csv(path: Path) -> list[dict[str, str]]:
if not path.exists():
raise FileNotFoundError(f"missing CSV file: {path}")
with path.open("r", encoding="utf-8", newline="") as handle:
rows = list(csv.DictReader(handle))
if not rows:
raise ValueError(f"CSV file contains no rows: {path}")
return rows
def _read_jsonl(path: Path | None) -> dict[tuple[str, str], dict[str, Any]]:
if path is None:
return {}
if not path.exists():
raise FileNotFoundError(f"missing JSONL file: {path}")
rows: dict[tuple[str, str], dict[str, Any]] = {}
with path.open("r", encoding="utf-8") as handle:
for line_number, line in enumerate(handle, start=1):
if not line.strip():
continue
payload = json.loads(line)
if not isinstance(payload, dict):
raise ValueError(f"{path}:{line_number}: JSONL row must be an object")
sample_id = str(payload.get("sample_id") or "")
line_id = str(payload.get("line_id") or payload.get("predicted_line_id") or "")
if line_id:
rows[(sample_id, line_id)] = payload
return rows
def _float(row: dict[str, Any], key: str, default: float = 0.0) -> float:
value = row.get(key, default)
try:
return float(value)
except (TypeError, ValueError):
return default
def _int(row: dict[str, Any], key: str, default: int = 0) -> int:
value = row.get(key, default)
try:
return int(value)
except (TypeError, ValueError):
return default
def _number_sequence(value: Any) -> list[float]:
if value in ("", None):
return []
if isinstance(value, (list, tuple)):
raw_items = value
elif isinstance(value, str):
try:
parsed = json.loads(value)
except json.JSONDecodeError:
try:
parsed = ast.literal_eval(value)
except (SyntaxError, ValueError):
return []
if not isinstance(parsed, (list, tuple)):
return []
raw_items = parsed
else:
return []
values: list[float] = []
for item in raw_items:
try:
values.append(float(item))
except (TypeError, ValueError):
return []
return values
def _bbox_geometry(value: Any) -> tuple[float | None, float | None, float | None, float | None]:
values = _number_sequence(value)
if len(values) != 4:
return None, None, None, None
x, y, width, height = values
if width < 0 or height < 0:
return None, None, None, None
return x, y, width, height
def _bbox_width_height(value: Any) -> tuple[float | None, float | None]:
_x, _y, width, height = _bbox_geometry(value)
return width, height
def _bbox_center(value: Any) -> tuple[float | None, float | None]:
x, y, width, height = _bbox_geometry(value)
if x is None or y is None or width is None or height is None:
return None, None
return x + width / 2.0, y + height / 2.0
def _xyxy_width_height(value: Any) -> tuple[float | None, float | None]:
values = _number_sequence(value)
if len(values) != 4:
return None, None
width = values[2] - values[0]
height = values[3] - values[1]
if width < 0 or height < 0:
return None, None
return width, height
def _ratio(numerator: float | None, denominator: float | None) -> float | None:
if numerator is None or denominator is None or denominator == 0:
return None
return numerator / denominator
def _bbox_iou(left_bbox: Any, right_bbox: Any) -> float:
left_x, left_y, left_w, left_h = _bbox_geometry(left_bbox)
right_x, right_y, right_w, right_h = _bbox_geometry(right_bbox)
if (
left_x is None
or left_y is None
or left_w is None
or left_h is None
or right_x is None
or right_y is None
or right_w is None
or right_h is None
):
return 0.0
left_x2 = left_x + left_w
left_y2 = left_y + left_h
right_x2 = right_x + right_w
right_y2 = right_y + right_h
inter_w = max(0.0, min(left_x2, right_x2) - max(left_x, right_x))
inter_h = max(0.0, min(left_y2, right_y2) - max(left_y, right_y))
inter_area = inter_w * inter_h
if inter_area <= 0:
return 0.0
union_area = left_w * left_h + right_w * right_h - inter_area
return inter_area / union_area if union_area > 0 else 0.0
def _joined_rows(
alignment_rows: list[dict[str, str]],
crop_rows: dict[tuple[str, str], dict[str, Any]],
*,
high_iou_threshold: float,
bad_codepoint_cer: float,
bad_grapheme_cer: float,
) -> list[dict[str, Any]]:
rows: list[dict[str, Any]] = []
for row in alignment_rows:
sample_id = row.get("sample_id") or ""
predicted_line_id = row.get("predicted_line_id") or ""
crop = crop_rows.get((sample_id, predicted_line_id), crop_rows.get(("", predicted_line_id), {}))
iou = _float(row, "iou")
cp_cer = _float(row, "text_cer_codepoint")
gr_cer = _float(row, "text_cer_grapheme")
kind = row.get("kind") or ""
crop_width, crop_height = _xyxy_width_height(crop.get("crop_xyxy"))
detected_width, detected_height = _bbox_width_height(crop.get("detected_bbox"))
reference_x, reference_y, reference_width, reference_height = _bbox_geometry(row.get("reference_bbox"))
predicted_x, predicted_y, predicted_width, predicted_height = _bbox_geometry(row.get("predicted_bbox"))
codepoint_ref_len = _int(row, "text_codepoint_ref_len")
grapheme_ref_len = _int(row, "text_grapheme_ref_len")
reference_center_x, reference_center_y = _bbox_center(row.get("reference_bbox"))
predicted_center_x, predicted_center_y = _bbox_center(row.get("predicted_bbox"))
bucket = kind
if kind == "matched":
good_geometry = iou >= high_iou_threshold
bad_text = cp_cer >= bad_codepoint_cer or gr_cer >= bad_grapheme_cer
if good_geometry and bad_text:
bucket = "matched_good_geometry_bad_text"
elif good_geometry:
bucket = "matched_good_geometry_acceptable_text"
elif bad_text:
bucket = "matched_low_geometry_bad_text"
else:
bucket = "matched_low_geometry_acceptable_text"
joined = {
"sample_id": sample_id,
"kind": kind,
"bucket": bucket,
"page_index": row.get("page_index") or "",
"reference_line_id": row.get("reference_line_id") or "",
"predicted_line_id": predicted_line_id,
"iou": iou,
"reference_bbox": row.get("reference_bbox") or "",
"predicted_bbox": row.get("predicted_bbox") or "",
"reference_text": row.get("reference_text") or "",
"predicted_text": row.get("predicted_text") or "",
"text_codepoint_edits": _int(row, "text_codepoint_edits"),
"text_codepoint_ref_len": codepoint_ref_len,
"text_cer_codepoint": cp_cer,
"text_grapheme_edits": _int(row, "text_grapheme_edits"),
"text_grapheme_ref_len": grapheme_ref_len,
"text_cer_grapheme": gr_cer,
"confidence": crop.get("confidence"),
"crop_path": crop.get("crop_path") or "",
"crop_sha256": crop.get("crop_sha256") or "",
"crop_xyxy": crop.get("crop_xyxy") or "",
"detected_bbox": crop.get("detected_bbox") or "",
"detected_area": crop.get("detected_area"),
"crop_width_px": crop_width,
"crop_height_px": crop_height,
"crop_aspect_ratio": _ratio(crop_width, crop_height),
"detected_width_px": detected_width,
"detected_height_px": detected_height,
"detected_aspect_ratio": _ratio(detected_width, detected_height),
"reference_width_px": reference_width,
"reference_height_px": reference_height,
"reference_x_px": reference_x,
"reference_y_px": reference_y,
"reference_center_x_px": reference_center_x,
"reference_center_y_px": reference_center_y,
"reference_aspect_ratio": _ratio(reference_width, reference_height),
"predicted_width_px": predicted_width,
"predicted_height_px": predicted_height,
"predicted_x_px": predicted_x,
"predicted_y_px": predicted_y,
"predicted_center_x_px": predicted_center_x,
"predicted_center_y_px": predicted_center_y,
"predicted_aspect_ratio": _ratio(predicted_width, predicted_height),
"crop_px_per_codepoint": _ratio(crop_width, float(codepoint_ref_len)),
"crop_px_per_grapheme": _ratio(crop_width, float(grapheme_ref_len)),
}
rows.append(joined)
return rows
def _predicted_overlap_candidates(rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
candidates: list[dict[str, Any]] = []
for row in rows:
if row.get("kind") not in {"matched", "prediction_unmatched"}:
continue
if not row.get("predicted_bbox"):
continue
candidates.append(row)
return candidates
def _annotate_unmatched_reference_prediction_overlaps(rows: list[dict[str, Any]]) -> None:
candidates = _predicted_overlap_candidates(rows)
for row in rows:
if row.get("kind") != "reference_unmatched":
continue
best: dict[str, Any] | None = None
best_iou = 0.0
for candidate in candidates:
if candidate.get("sample_id") != row.get("sample_id") or candidate.get("page_index") != row.get("page_index"):
continue
iou = _bbox_iou(row.get("reference_bbox"), candidate.get("predicted_bbox"))
if iou > best_iou:
best_iou = iou
best = candidate
row["best_any_prediction_iou"] = best_iou
row["best_any_prediction_line_id"] = best.get("predicted_line_id") if best is not None else ""
row["best_any_prediction_kind"] = best.get("kind") if best is not None else ""
row["best_any_prediction_bbox"] = best.get("predicted_bbox") if best is not None else ""
row["best_any_prediction_text"] = best.get("predicted_text") if best is not None else ""
row["best_any_prediction_confidence"] = best.get("confidence") if best is not None else None
def _bucket_counts(rows: list[dict[str, Any]]) -> dict[str, int]:
counts: dict[str, int] = {}
for row in rows:
bucket = str(row.get("bucket") or "unknown")
counts[bucket] = counts.get(bucket, 0) + 1
return dict(sorted(counts.items()))
def _bucket_stats(rows: list[dict[str, Any]]) -> dict[str, dict[str, float | int | None]]:
buckets = sorted({str(row.get("bucket") or "unknown") for row in rows})
stats: dict[str, dict[str, float | int | None]] = {}
for bucket in buckets:
bucket_rows = [row for row in rows if row.get("bucket") == bucket]
stats[bucket] = {
"count": len(bucket_rows),
"mean_iou": _mean(bucket_rows, "iou"),
"mean_confidence": _mean(bucket_rows, "confidence"),
"mean_codepoint_ref_len": _mean(bucket_rows, "text_codepoint_ref_len"),
"mean_grapheme_ref_len": _mean(bucket_rows, "text_grapheme_ref_len"),
"mean_codepoint_cer": _mean(bucket_rows, "text_cer_codepoint"),
"mean_grapheme_cer": _mean(bucket_rows, "text_cer_grapheme"),
"mean_crop_width_px": _mean(bucket_rows, "crop_width_px"),
"mean_crop_height_px": _mean(bucket_rows, "crop_height_px"),
"mean_crop_aspect_ratio": _mean(bucket_rows, "crop_aspect_ratio"),
"mean_reference_x_px": _mean(bucket_rows, "reference_x_px"),
"mean_reference_y_px": _mean(bucket_rows, "reference_y_px"),
"mean_reference_width_px": _mean(bucket_rows, "reference_width_px"),
"mean_reference_height_px": _mean(bucket_rows, "reference_height_px"),
"mean_reference_aspect_ratio": _mean(bucket_rows, "reference_aspect_ratio"),
"mean_predicted_x_px": _mean(bucket_rows, "predicted_x_px"),
"mean_predicted_y_px": _mean(bucket_rows, "predicted_y_px"),
"mean_predicted_width_px": _mean(bucket_rows, "predicted_width_px"),
"mean_predicted_height_px": _mean(bucket_rows, "predicted_height_px"),
"mean_predicted_aspect_ratio": _mean(bucket_rows, "predicted_aspect_ratio"),
"mean_crop_px_per_codepoint": _mean(bucket_rows, "crop_px_per_codepoint"),
"mean_crop_px_per_grapheme": _mean(bucket_rows, "crop_px_per_grapheme"),
}
return stats
def _mean(rows: list[dict[str, Any]], key: str) -> float | None:
values = [_float(row, key) for row in rows if row.get(key) not in ("", None)]
return sum(values) / len(values) if values else None
def _crop_width_band(width: Any) -> str:
if width in ("", None):
return "missing"
value = _float({"width": width}, "width", -1.0)
if value < 0:
return "missing"
if value <= 256:
return "000-256"
if value <= 384:
return "257-384"
if value <= 512:
return "385-512"
if value <= 768:
return "513-768"
return "769-plus"
def _crop_width_band_stats(rows: list[dict[str, Any]]) -> dict[str, dict[str, float | int | None]]:
matched = [row for row in rows if row.get("kind") == "matched"]
bands = sorted({_crop_width_band(row.get("crop_width_px")) for row in matched})
stats: dict[str, dict[str, float | int | None]] = {}
for band in bands:
band_rows = [row for row in matched if _crop_width_band(row.get("crop_width_px")) == band]
stats[band] = {
"count": len(band_rows),
"mean_iou": _mean(band_rows, "iou"),
"mean_confidence": _mean(band_rows, "confidence"),
"mean_codepoint_ref_len": _mean(band_rows, "text_codepoint_ref_len"),
"mean_codepoint_cer": _mean(band_rows, "text_cer_codepoint"),
"mean_grapheme_cer": _mean(band_rows, "text_cer_grapheme"),
"mean_crop_aspect_ratio": _mean(band_rows, "crop_aspect_ratio"),
}
return stats
def _text_codepoint_len(value: Any) -> int:
if value in ("", None):
return 0
return len(str(value))
def _codepoint_error_mass(rows: list[dict[str, Any]]) -> dict[str, Any]:
sources = {
"matched_good_geometry_bad_text": 0,
"matched_good_geometry_acceptable_text": 0,
"matched_low_geometry_bad_text": 0,
"matched_low_geometry_acceptable_text": 0,
"missed_reference_text": 0,
"extra_prediction_text": 0,
"other": 0,
}
for row in rows:
kind = row.get("kind")
if kind == "matched":
bucket = str(row.get("bucket") or "other")
sources[bucket if bucket in sources else "other"] += _int(row, "text_codepoint_edits")
elif kind == "reference_unmatched":
ref_len = _int(row, "text_codepoint_ref_len")
sources["missed_reference_text"] += ref_len if ref_len > 0 else _text_codepoint_len(row.get("reference_text"))
elif kind == "prediction_unmatched":
sources["extra_prediction_text"] += _text_codepoint_len(row.get("predicted_text"))
else:
sources["other"] += _int(row, "text_codepoint_edits")
total = sum(sources.values())
fractions = {key: value / total if total else 0.0 for key, value in sources.items()}
dominant_source = max(sources, key=lambda key: (sources[key], key)) if total else None
return {
"total": total,
"by_source": sources,
"fractions": fractions,
"dominant_source": dominant_source,
}
def _spatial_group_key(row: dict[str, Any]) -> tuple[str, str]:
return str(row.get("sample_id") or ""), str(row.get("page_index") or "")
def _reference_spatial_context(rows: list[dict[str, Any]]) -> dict[tuple[str, str], dict[str, float]]:
context: dict[tuple[str, str], dict[str, float]] = {}
for row in rows:
if row.get("kind") not in {"matched", "reference_unmatched"}:
continue
center_x = row.get("reference_center_x_px")
center_y = row.get("reference_center_y_px")
if center_x in ("", None) or center_y in ("", None):
continue
x = float(center_x)
y = float(center_y)
key = _spatial_group_key(row)
if key not in context:
context[key] = {"min_x": x, "max_x": x, "min_y": y, "max_y": y}
continue
item = context[key]
item["min_x"] = min(item["min_x"], x)
item["max_x"] = max(item["max_x"], x)
item["min_y"] = min(item["min_y"], y)
item["max_y"] = max(item["max_y"], y)
return context
def _axis_band(value: Any, *, minimum: float, maximum: float, labels: tuple[str, str, str]) -> str:
if value in ("", None):
return "unknown"
span = maximum - minimum
if span <= 0:
return labels[1]
ratio = (float(value) - minimum) / span
if ratio < 1 / 3:
return labels[0]
if ratio < 2 / 3:
return labels[1]
return labels[2]
def _empty_spatial_bucket() -> dict[str, float | int | None]:
return {
"reference_count": 0,
"unmatched_reference_count": 0,
"unmatched_reference_fraction": 0.0,
"codepoint_error_mass": 0,
"mean_reference_x_px": None,
"mean_reference_y_px": None,
"mean_reference_width_px": None,
"mean_reference_height_px": None,
}
def _finalize_spatial_bucket(rows: list[dict[str, Any]], unmatched_rows: list[dict[str, Any]]) -> dict[str, float | int | None]:
reference_count = len(rows)
unmatched_count = len(unmatched_rows)
return {
"reference_count": reference_count,
"unmatched_reference_count": unmatched_count,
"unmatched_reference_fraction": unmatched_count / reference_count if reference_count else 0.0,
"codepoint_error_mass": sum(
_int(row, "text_codepoint_ref_len")
if _int(row, "text_codepoint_ref_len") > 0
else _text_codepoint_len(row.get("reference_text"))
for row in unmatched_rows
),
"mean_reference_x_px": _mean(unmatched_rows, "reference_x_px"),
"mean_reference_y_px": _mean(unmatched_rows, "reference_y_px"),
"mean_reference_width_px": _mean(unmatched_rows, "reference_width_px"),
"mean_reference_height_px": _mean(unmatched_rows, "reference_height_px"),
}
def _reference_spatial_stats(rows: list[dict[str, Any]]) -> dict[str, Any]:
context = _reference_spatial_context(rows)
reference_rows: list[dict[str, Any]] = []
for row in rows:
if row.get("kind") not in {"matched", "reference_unmatched"}:
continue
group = context.get(_spatial_group_key(row))
if group is None:
continue
y_band = _axis_band(
row.get("reference_center_y_px"),
minimum=group["min_y"],
maximum=group["max_y"],
labels=("top", "middle", "bottom"),
)
x_band = _axis_band(
row.get("reference_center_x_px"),
minimum=group["min_x"],
maximum=group["max_x"],
labels=("left", "center", "right"),
)
enriched = dict(row)
enriched["reference_y_band"] = y_band
enriched["reference_x_band"] = x_band
enriched["reference_xy_band"] = f"{y_band}/{x_band}"
reference_rows.append(enriched)
def grouped(field: str, ordered_keys: tuple[str, ...] | None = None) -> dict[str, dict[str, float | int | None]]:
keys = list(ordered_keys or tuple(sorted({str(row[field]) for row in reference_rows})))
stats: dict[str, dict[str, float | int | None]] = {}
for key in keys:
bucket_rows = [row for row in reference_rows if row[field] == key]
unmatched_rows = [row for row in bucket_rows if row.get("kind") == "reference_unmatched"]
stats[key] = _finalize_spatial_bucket(bucket_rows, unmatched_rows) if bucket_rows else _empty_spatial_bucket()
return stats
xy_keys = tuple(
f"{y}/{x}"
for y in ("top", "middle", "bottom")
for x in ("left", "center", "right")
)
by_y_band = grouped("reference_y_band", ("top", "middle", "bottom", "unknown"))
by_x_band = grouped("reference_x_band", ("left", "center", "right", "unknown"))
by_xy_band = grouped("reference_xy_band", (*xy_keys, "unknown/unknown"))
top_unmatched_bands = sorted(
(
{"band": band, **stats}
for band, stats in by_xy_band.items()
if int(stats["unmatched_reference_count"] or 0) > 0
),
key=lambda row: (
int(row["codepoint_error_mass"] or 0),
int(row["unmatched_reference_count"] or 0),
float(row["unmatched_reference_fraction"] or 0.0),
str(row["band"]),
),
reverse=True,
)
return {
"reference_count": len(reference_rows),
"unmatched_reference_count": sum(row.get("kind") == "reference_unmatched" for row in reference_rows),
"by_y_band": by_y_band,
"by_x_band": by_x_band,
"by_xy_band": by_xy_band,
"top_unmatched_bands": top_unmatched_bands,
}
def _unmatched_reference_overlap_stats(rows: list[dict[str, Any]]) -> dict[str, Any]:
unmatched = [row for row in rows if row.get("kind") == "reference_unmatched"]
source_counts: dict[str, int] = {}
for row in unmatched:
source = str(row.get("best_any_prediction_kind") or "none")
source_counts[source] = source_counts.get(source, 0) + 1
return {
"reference_unmatched_count": len(unmatched),
"best_any_prediction_iou_ge_010": sum(_float(row, "best_any_prediction_iou") >= 0.10 for row in unmatched),
"best_any_prediction_iou_ge_030": sum(_float(row, "best_any_prediction_iou") >= 0.30 for row in unmatched),
"best_any_prediction_iou_ge_050": sum(_float(row, "best_any_prediction_iou") >= 0.50 for row in unmatched),
"best_any_prediction_iou_ge_085": sum(_float(row, "best_any_prediction_iou") >= 0.85 for row in unmatched),
"best_any_prediction_source_counts": dict(sorted(source_counts.items())),
"mean_best_any_prediction_iou": _mean(unmatched, "best_any_prediction_iou"),
}
def _top_failures(rows: list[dict[str, Any]], limit: int) -> list[dict[str, Any]]:
candidates = [row for row in rows if row["kind"] == "matched"]
candidates.sort(
key=lambda row: (
_int(row, "text_codepoint_edits"),
_float(row, "text_cer_grapheme"),
_float(row, "text_cer_codepoint"),
),
reverse=True,
)
return candidates[:limit]
def _top_unmatched_references(rows: list[dict[str, Any]], limit: int) -> list[dict[str, Any]]:
candidates = [row for row in rows if row["kind"] == "reference_unmatched"]
candidates.sort(
key=lambda row: (
_int(row, "text_codepoint_ref_len"),
_float(row, "best_any_prediction_iou"),
_float(row, "reference_width_px") * _float(row, "reference_height_px"),
_float(row, "reference_y_px"),
_float(row, "reference_x_px"),
),
reverse=True,
)
return candidates[:limit]
def _top_unmatched_predictions(rows: list[dict[str, Any]], limit: int) -> list[dict[str, Any]]:
candidates = [row for row in rows if row["kind"] == "prediction_unmatched"]
candidates.sort(
key=lambda row: (
_float(row, "confidence", -1.0),
len(str(row.get("predicted_text") or "")),
_float(row, "predicted_width_px") * _float(row, "predicted_height_px"),
),
reverse=True,
)
return candidates[:limit]
def _near_miss_unmatched_pairs(rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
references = [row for row in rows if row["kind"] == "reference_unmatched"]
predictions = [row for row in rows if row["kind"] == "prediction_unmatched"]
pairs: list[dict[str, Any]] = []
for reference in references:
sample_id = reference.get("sample_id")
page_index = reference.get("page_index")
candidates = [
prediction
for prediction in predictions
if prediction.get("sample_id") == sample_id and prediction.get("page_index") == page_index
]
best_prediction: dict[str, Any] | None = None
best_iou = 0.0
for prediction in candidates:
iou = _bbox_iou(reference.get("reference_bbox"), prediction.get("predicted_bbox"))
if iou > best_iou:
best_iou = iou
best_prediction = prediction
if best_prediction is None:
continue
pairs.append(
{
"sample_id": sample_id,
"page_index": page_index,
"reference_line_id": reference.get("reference_line_id") or "",
"predicted_line_id": best_prediction.get("predicted_line_id") or "",
"iou": best_iou,
"reference_bbox": reference.get("reference_bbox") or "",
"predicted_bbox": best_prediction.get("predicted_bbox") or "",
"reference_text": reference.get("reference_text") or "",
"predicted_text": best_prediction.get("predicted_text") or "",
"confidence": best_prediction.get("confidence"),
"reference_width_px": reference.get("reference_width_px"),
"reference_height_px": reference.get("reference_height_px"),
"predicted_width_px": best_prediction.get("predicted_width_px"),
"predicted_height_px": best_prediction.get("predicted_height_px"),
"crop_path": best_prediction.get("crop_path") or "",
"crop_sha256": best_prediction.get("crop_sha256") or "",
}
)
pairs.sort(key=lambda row: _float(row, "iou"), reverse=True)
return pairs
def _write_rows(path: Path, rows: list[dict[str, Any]]) -> None:
fields = [
"sample_id",
"kind",
"bucket",
"page_index",
"reference_line_id",
"predicted_line_id",
"iou",
"text_codepoint_edits",
"text_codepoint_ref_len",
"text_cer_codepoint",
"text_grapheme_edits",
"text_grapheme_ref_len",
"text_cer_grapheme",
"confidence",
"crop_path",
"crop_sha256",
"reference_bbox",
"predicted_bbox",
"crop_xyxy",
"detected_bbox",
"detected_area",
"crop_width_px",
"crop_height_px",
"crop_aspect_ratio",
"detected_width_px",
"detected_height_px",
"detected_aspect_ratio",
"reference_width_px",
"reference_height_px",
"reference_x_px",
"reference_y_px",
"reference_center_x_px",
"reference_center_y_px",
"reference_aspect_ratio",
"predicted_width_px",
"predicted_height_px",
"predicted_x_px",
"predicted_y_px",
"predicted_center_x_px",
"predicted_center_y_px",
"predicted_aspect_ratio",
"best_any_prediction_iou",
"best_any_prediction_line_id",
"best_any_prediction_kind",
"best_any_prediction_bbox",
"best_any_prediction_confidence",
"crop_px_per_codepoint",
"crop_px_per_grapheme",
"reference_text",
"predicted_text",
"best_any_prediction_text",
]
with path.open("w", encoding="utf-8", newline="") as handle:
writer = csv.DictWriter(handle, fieldnames=fields)
writer.writeheader()
for row in rows:
payload = dict(row)
for key in ("crop_xyxy", "detected_bbox"):
if isinstance(payload.get(key), list):
payload[key] = json.dumps(payload[key], ensure_ascii=False)
writer.writerow({field: payload.get(field, "") for field in fields})
def _write_near_miss_pairs(path: Path, rows: list[dict[str, Any]]) -> None:
fields = [
"sample_id",
"page_index",
"reference_line_id",
"predicted_line_id",
"iou",
"confidence",
"reference_bbox",
"predicted_bbox",
"reference_width_px",
"reference_height_px",
"best_any_prediction_iou",
"best_any_prediction_line_id",
"best_any_prediction_kind",
"best_any_prediction_bbox",
"best_any_prediction_confidence",
"predicted_width_px",
"predicted_height_px",
"crop_path",
"crop_sha256",
"reference_text",
"predicted_text",
]
with path.open("w", encoding="utf-8", newline="") as handle:
writer = csv.DictWriter(handle, fieldnames=fields)
writer.writeheader()
for row in rows:
writer.writerow({field: row.get(field, "") for field in fields})
def _write_report(path: Path, result: dict[str, Any]) -> None:
summary = result["summary"]
buckets = summary["bucket_counts"]
lines = [
"# Page Line Failure Analysis",
"",
"This report joins page line-alignment rows with crop provenance. It is diagnostic evidence, not a claim-safe score.",
"",
f"- line alignment: `{result['line_alignment_csv']}`",
f"- crop manifest: `{result.get('crop_manifest') or ''}`",
f"- rows: {summary['row_count']}",
f"- matched rows: {summary['matched_count']}",
f"- unmatched references: {summary['reference_unmatched_count']}",
f"- unmatched predictions: {summary['prediction_unmatched_count']}",
f"- high-IoU threshold: {result['thresholds']['high_iou']}",
f"- bad codepoint CER threshold: {result['thresholds']['bad_codepoint_cer']}",
f"- bad grapheme CER threshold: {result['thresholds']['bad_grapheme_cer']}",
f"- high-IoU bad-text matched rows: {buckets.get('matched_good_geometry_bad_text', 0)}",
f"- high-IoU acceptable-text matched rows: {buckets.get('matched_good_geometry_acceptable_text', 0)}",
f"- mean matched IoU: {summary['matched_mean_iou']:.6f}",
f"- mean matched codepoint CER: {summary['matched_mean_codepoint_cer']:.6f}",
f"- mean matched grapheme CER: {summary['matched_mean_grapheme_cer']:.6f}",
f"- mean unmatched-reference y: {_format_optional_float(summary['bucket_stats'].get('reference_unmatched', {}).get('mean_reference_y_px'))}",
f"- mean unmatched-prediction confidence: {_format_optional_float(summary['bucket_stats'].get('prediction_unmatched', {}).get('mean_confidence'))}",
f"- unmatched reference/prediction near-miss pairs at IoU >= 0.30: {summary['near_miss_unmatched_pair_count_iou_ge_030']}",
f"- unmatched reference/prediction near-miss pairs at IoU >= 0.50: {summary['near_miss_unmatched_pair_count_iou_ge_050']}",
f"- unmatched references with any prediction IoU >= 0.30: {summary['unmatched_reference_overlap_stats']['best_any_prediction_iou_ge_030']}",
f"- unmatched references with any prediction IoU >= 0.50: {summary['unmatched_reference_overlap_stats']['best_any_prediction_iou_ge_050']}",
f"- unmatched references with no overlapping prediction: {summary['unmatched_reference_overlap_stats']['best_any_prediction_source_counts'].get('none', 0)}",
f"- dominant codepoint error source: {summary['codepoint_error_mass']['dominant_source'] or 'n/a'}",
f"- total attributed codepoint error mass: {summary['codepoint_error_mass']['total']}",
"",
"## Geometry Signal",
"",
f"- mean high-IoU bad-text crop width: {_format_optional_float(summary['bucket_stats'].get('matched_good_geometry_bad_text', {}).get('mean_crop_width_px'))}",
f"- mean high-IoU acceptable-text crop width: {_format_optional_float(summary['bucket_stats'].get('matched_good_geometry_acceptable_text', {}).get('mean_crop_width_px'))}",
f"- mean high-IoU bad-text crop aspect ratio: {_format_optional_float(summary['bucket_stats'].get('matched_good_geometry_bad_text', {}).get('mean_crop_aspect_ratio'))}",
f"- mean high-IoU acceptable-text crop aspect ratio: {_format_optional_float(summary['bucket_stats'].get('matched_good_geometry_acceptable_text', {}).get('mean_crop_aspect_ratio'))}",
f"- mean unmatched-reference width: {_format_optional_float(summary['bucket_stats'].get('reference_unmatched', {}).get('mean_reference_width_px'))}",
f"- mean unmatched-reference height: {_format_optional_float(summary['bucket_stats'].get('reference_unmatched', {}).get('mean_reference_height_px'))}",
f"- mean unmatched-prediction width: {_format_optional_float(summary['bucket_stats'].get('prediction_unmatched', {}).get('mean_predicted_width_px'))}",
f"- mean unmatched-prediction height: {_format_optional_float(summary['bucket_stats'].get('prediction_unmatched', {}).get('mean_predicted_height_px'))}",
"",
"## Missed Reference Spatial Bands",
"",
]
spatial = summary["unmatched_reference_spatial_stats"]
for band, stats in spatial["by_y_band"].items():
lines.append(
"- "
f"{band}: {stats['unmatched_reference_count']}/{stats['reference_count']} unmatched "
f"({float(stats['unmatched_reference_fraction']):.3f}), "
f"error mass {stats['codepoint_error_mass']}"
)
lines.extend(["", "By horizontal band:", ""])
for band, stats in spatial["by_x_band"].items():
lines.append(
"- "
f"{band}: {stats['unmatched_reference_count']}/{stats['reference_count']} unmatched "
f"({float(stats['unmatched_reference_fraction']):.3f}), "
f"error mass {stats['codepoint_error_mass']}"
)
lines.extend(["", "Top combined bands:", ""])
for row in spatial["top_unmatched_bands"][:5]:
lines.append(
"- "
f"{row['band']}: {row['unmatched_reference_count']}/{row['reference_count']} unmatched "
f"({float(row['unmatched_reference_fraction']):.3f}), "
f"error mass {row['codepoint_error_mass']}"
)
lines.extend([
"",
"## Error Mass",
"",
])
for source, count in summary["codepoint_error_mass"]["by_source"].items():
fraction = summary["codepoint_error_mass"]["fractions"].get(source, 0.0)
lines.append(f"- {source}: {count} ({fraction:.3f})")
lines.extend([
"",
"## Top Failures",
"",
])
for row in result["top_failures"][:10]:
lines.append(
"- "
f"{row['reference_line_id']} -> {row['predicted_line_id']}: "
f"IoU {row['iou']:.3f}, cp CER {row['text_cer_codepoint']:.3f}, "
f"gr CER {row['text_cer_grapheme']:.3f}, crop `{row['crop_path']}`"
)
lines.extend(["", "## Top Unmatched References", ""])
for row in result.get("top_unmatched_references", [])[:10]:
lines.append(
"- "
f"{row['reference_line_id']}: ref len {row['text_codepoint_ref_len']}, "
f"bbox `{row['reference_bbox']}`, "
f"best any prediction IoU {_format_optional_float(row.get('best_any_prediction_iou'))} "
f"({row.get('best_any_prediction_kind') or 'none'} {row.get('best_any_prediction_line_id') or ''})"
)
lines.extend(["", "## Top Unmatched Predictions", ""])
for row in result.get("top_unmatched_predictions", [])[:10]:
lines.append(
"- "
f"{row['predicted_line_id']}: confidence {_format_optional_float(row.get('confidence'))}, "
f"bbox `{row['predicted_bbox']}`, text `{row['predicted_text']}`"
)
lines.extend(["", "## Top Unmatched Near Misses", ""])
for row in result.get("near_miss_unmatched_pairs", [])[:10]:
lines.append(
"- "
f"{row['reference_line_id']} -> {row['predicted_line_id']}: "
f"IoU {row['iou']:.3f}, confidence {_format_optional_float(row.get('confidence'))}, "
f"crop `{row['crop_path']}`"
)
path.write_text("\n".join(lines) + "\n", encoding="utf-8")
def _format_optional_float(value: float | int | None) -> str:
if value is None:
return "n/a"
return f"{float(value):.6f}"
def analyze(args: argparse.Namespace) -> dict[str, Any]:
if args.high_iou_threshold <= 0 or args.high_iou_threshold > 1:
raise ValueError("--high-iou-threshold must be in (0, 1]")
if args.bad_codepoint_cer < 0 or args.bad_grapheme_cer < 0:
raise ValueError("bad CER thresholds must be non-negative")
args.out.mkdir(parents=True, exist_ok=True)
alignment_rows = _read_csv(args.line_alignment_csv)
crop_rows = _read_jsonl(args.crop_manifest)
joined = _joined_rows(
alignment_rows,
crop_rows,
high_iou_threshold=args.high_iou_threshold,
bad_codepoint_cer=args.bad_codepoint_cer,
bad_grapheme_cer=args.bad_grapheme_cer,
)
_annotate_unmatched_reference_prediction_overlaps(joined)
matched = [row for row in joined if row["kind"] == "matched"]
top = _top_failures(joined, args.top_k)
top_unmatched_references = _top_unmatched_references(joined, args.top_k)
top_unmatched_predictions = _top_unmatched_predictions(joined, args.top_k)
near_miss_unmatched_pairs = _near_miss_unmatched_pairs(joined)
bucket_counts = _bucket_counts(joined)
bucket_stats = _bucket_stats(joined)
crop_width_band_stats = _crop_width_band_stats(joined)
codepoint_error_mass = _codepoint_error_mass(joined)
unmatched_reference_spatial_stats = _reference_spatial_stats(joined)
unmatched_reference_overlap_stats = _unmatched_reference_overlap_stats(joined)
result = {
"created_at": datetime.now(timezone.utc).isoformat(),
"machine": platform.node(),
"tool": "tools/analyze_page_line_failures.py",
"line_alignment_csv": str(args.line_alignment_csv),
"line_alignment_csv_sha256": sha256_file(args.line_alignment_csv),
"crop_manifest": str(args.crop_manifest) if args.crop_manifest else None,
"crop_manifest_sha256": sha256_file(args.crop_manifest) if args.crop_manifest else None,
"thresholds": {
"high_iou": args.high_iou_threshold,
"bad_codepoint_cer": args.bad_codepoint_cer,
"bad_grapheme_cer": args.bad_grapheme_cer,
},
"summary": {
"row_count": len(joined),
"matched_count": len(matched),
"reference_unmatched_count": sum(row["kind"] == "reference_unmatched" for row in joined),
"prediction_unmatched_count": sum(row["kind"] == "prediction_unmatched" for row in joined),
"bucket_counts": bucket_counts,
"bucket_stats": bucket_stats,
"crop_width_band_stats": crop_width_band_stats,
"codepoint_error_mass": codepoint_error_mass,
"unmatched_reference_spatial_stats": unmatched_reference_spatial_stats,
"unmatched_reference_overlap_stats": unmatched_reference_overlap_stats,
"matched_mean_iou": _mean(matched, "iou") or 0.0,
"matched_mean_codepoint_cer": _mean(matched, "text_cer_codepoint") or 0.0,
"matched_mean_grapheme_cer": _mean(matched, "text_cer_grapheme") or 0.0,
"matched_good_geometry_bad_text_fraction": bucket_counts.get("matched_good_geometry_bad_text", 0)
/ max(len(matched), 1),
"near_miss_unmatched_pair_count_iou_ge_030": sum(
_float(row, "iou") >= 0.30 for row in near_miss_unmatched_pairs
),
"near_miss_unmatched_pair_count_iou_ge_050": sum(
_float(row, "iou") >= 0.50 for row in near_miss_unmatched_pairs
),
},
"top_failures": top,
"top_unmatched_references": top_unmatched_references,
"top_unmatched_predictions": top_unmatched_predictions,
"near_miss_unmatched_pairs": near_miss_unmatched_pairs[: args.top_k],
}
rows_path = args.out / "line-failures.csv"
top_path = args.out / "top-line-failures.csv"
top_unmatched_references_path = args.out / "top-unmatched-references.csv"
top_unmatched_predictions_path = args.out / "top-unmatched-predictions.csv"
near_miss_unmatched_pairs_path = args.out / "near-miss-unmatched-pairs.csv"
result_path = args.out / "line-failure-analysis.json"
report_path = args.out / "line-failure-analysis.md"
_write_rows(rows_path, joined)
_write_rows(top_path, top)
_write_rows(top_unmatched_references_path, top_unmatched_references)
_write_rows(top_unmatched_predictions_path, top_unmatched_predictions)
_write_near_miss_pairs(near_miss_unmatched_pairs_path, near_miss_unmatched_pairs)
result["rows_csv"] = str(rows_path)
result["top_failures_csv"] = str(top_path)
result["top_unmatched_references_csv"] = str(top_unmatched_references_path)
result["top_unmatched_predictions_csv"] = str(top_unmatched_predictions_path)
result["near_miss_unmatched_pairs_csv"] = str(near_miss_unmatched_pairs_path)
result["result_path"] = str(result_path)
result["report_path"] = str(report_path)
result_path.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
_write_report(report_path, result)
return result
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--line-alignment-csv", type=Path, required=True)
parser.add_argument("--crop-manifest", type=Path)
parser.add_argument("--out", type=Path, required=True)
parser.add_argument("--high-iou-threshold", type=float, default=0.85)
parser.add_argument("--bad-codepoint-cer", type=float, default=0.30)
parser.add_argument("--bad-grapheme-cer", type=float, default=0.50)
parser.add_argument("--top-k", type=int, default=50)
args = parser.parse_args(argv)
result = analyze(args)
summary = result["summary"]
print(
f"wrote {result['result_path']} matched={summary['matched_count']} "
f"good_geometry_bad_text={summary['bucket_counts'].get('matched_good_geometry_bad_text', 0)} "
f"dominant_error_source={summary['codepoint_error_mass']['dominant_source'] or 'n/a'}"
)
return 0
if __name__ == "__main__":
raise SystemExit(main())