#!/usr/bin/env python3 """Analyze page OCR line-alignment failures with crop provenance. The page scorer already emits `line-alignment.csv`. This diagnostic joins those alignment rows to an image-line crop manifest when available, then writes ranked failure examples and aggregate buckets that distinguish geometry failures from recognizer/crop-quality failures on well-aligned boxes. """ from __future__ import annotations import argparse import csv import json import platform import ast from datetime import datetime, timezone from pathlib import Path from typing import Any from ocrtech.manifest import sha256_file def _read_csv(path: Path) -> list[dict[str, str]]: if not path.exists(): raise FileNotFoundError(f"missing CSV file: {path}") with path.open("r", encoding="utf-8", newline="") as handle: rows = list(csv.DictReader(handle)) if not rows: raise ValueError(f"CSV file contains no rows: {path}") return rows def _read_jsonl(path: Path | None) -> dict[tuple[str, str], dict[str, Any]]: if path is None: return {} if not path.exists(): raise FileNotFoundError(f"missing JSONL file: {path}") rows: dict[tuple[str, str], dict[str, Any]] = {} with path.open("r", encoding="utf-8") as handle: for line_number, line in enumerate(handle, start=1): if not line.strip(): continue payload = json.loads(line) if not isinstance(payload, dict): raise ValueError(f"{path}:{line_number}: JSONL row must be an object") sample_id = str(payload.get("sample_id") or "") line_id = str(payload.get("line_id") or payload.get("predicted_line_id") or "") if line_id: rows[(sample_id, line_id)] = payload return rows def _float(row: dict[str, Any], key: str, default: float = 0.0) -> float: value = row.get(key, default) try: return float(value) except (TypeError, ValueError): return default def _int(row: dict[str, Any], key: str, default: int = 0) -> int: value = row.get(key, default) try: return int(value) except (TypeError, ValueError): return default def _number_sequence(value: Any) -> list[float]: if value in ("", None): return [] if isinstance(value, (list, tuple)): raw_items = value elif isinstance(value, str): try: parsed = json.loads(value) except json.JSONDecodeError: try: parsed = ast.literal_eval(value) except (SyntaxError, ValueError): return [] if not isinstance(parsed, (list, tuple)): return [] raw_items = parsed else: return [] values: list[float] = [] for item in raw_items: try: values.append(float(item)) except (TypeError, ValueError): return [] return values def _bbox_geometry(value: Any) -> tuple[float | None, float | None, float | None, float | None]: values = _number_sequence(value) if len(values) != 4: return None, None, None, None x, y, width, height = values if width < 0 or height < 0: return None, None, None, None return x, y, width, height def _bbox_width_height(value: Any) -> tuple[float | None, float | None]: _x, _y, width, height = _bbox_geometry(value) return width, height def _bbox_center(value: Any) -> tuple[float | None, float | None]: x, y, width, height = _bbox_geometry(value) if x is None or y is None or width is None or height is None: return None, None return x + width / 2.0, y + height / 2.0 def _xyxy_width_height(value: Any) -> tuple[float | None, float | None]: values = _number_sequence(value) if len(values) != 4: return None, None width = values[2] - values[0] height = values[3] - values[1] if width < 0 or height < 0: return None, None return width, height def _ratio(numerator: float | None, denominator: float | None) -> float | None: if numerator is None or denominator is None or denominator == 0: return None return numerator / denominator def _bbox_iou(left_bbox: Any, right_bbox: Any) -> float: left_x, left_y, left_w, left_h = _bbox_geometry(left_bbox) right_x, right_y, right_w, right_h = _bbox_geometry(right_bbox) if ( left_x is None or left_y is None or left_w is None or left_h is None or right_x is None or right_y is None or right_w is None or right_h is None ): return 0.0 left_x2 = left_x + left_w left_y2 = left_y + left_h right_x2 = right_x + right_w right_y2 = right_y + right_h inter_w = max(0.0, min(left_x2, right_x2) - max(left_x, right_x)) inter_h = max(0.0, min(left_y2, right_y2) - max(left_y, right_y)) inter_area = inter_w * inter_h if inter_area <= 0: return 0.0 union_area = left_w * left_h + right_w * right_h - inter_area return inter_area / union_area if union_area > 0 else 0.0 def _joined_rows( alignment_rows: list[dict[str, str]], crop_rows: dict[tuple[str, str], dict[str, Any]], *, high_iou_threshold: float, bad_codepoint_cer: float, bad_grapheme_cer: float, ) -> list[dict[str, Any]]: rows: list[dict[str, Any]] = [] for row in alignment_rows: sample_id = row.get("sample_id") or "" predicted_line_id = row.get("predicted_line_id") or "" crop = crop_rows.get((sample_id, predicted_line_id), crop_rows.get(("", predicted_line_id), {})) iou = _float(row, "iou") cp_cer = _float(row, "text_cer_codepoint") gr_cer = _float(row, "text_cer_grapheme") kind = row.get("kind") or "" crop_width, crop_height = _xyxy_width_height(crop.get("crop_xyxy")) detected_width, detected_height = _bbox_width_height(crop.get("detected_bbox")) reference_x, reference_y, reference_width, reference_height = _bbox_geometry(row.get("reference_bbox")) predicted_x, predicted_y, predicted_width, predicted_height = _bbox_geometry(row.get("predicted_bbox")) codepoint_ref_len = _int(row, "text_codepoint_ref_len") grapheme_ref_len = _int(row, "text_grapheme_ref_len") reference_center_x, reference_center_y = _bbox_center(row.get("reference_bbox")) predicted_center_x, predicted_center_y = _bbox_center(row.get("predicted_bbox")) bucket = kind if kind == "matched": good_geometry = iou >= high_iou_threshold bad_text = cp_cer >= bad_codepoint_cer or gr_cer >= bad_grapheme_cer if good_geometry and bad_text: bucket = "matched_good_geometry_bad_text" elif good_geometry: bucket = "matched_good_geometry_acceptable_text" elif bad_text: bucket = "matched_low_geometry_bad_text" else: bucket = "matched_low_geometry_acceptable_text" joined = { "sample_id": sample_id, "kind": kind, "bucket": bucket, "page_index": row.get("page_index") or "", "reference_line_id": row.get("reference_line_id") or "", "predicted_line_id": predicted_line_id, "iou": iou, "reference_bbox": row.get("reference_bbox") or "", "predicted_bbox": row.get("predicted_bbox") or "", "reference_text": row.get("reference_text") or "", "predicted_text": row.get("predicted_text") or "", "text_codepoint_edits": _int(row, "text_codepoint_edits"), "text_codepoint_ref_len": codepoint_ref_len, "text_cer_codepoint": cp_cer, "text_grapheme_edits": _int(row, "text_grapheme_edits"), "text_grapheme_ref_len": grapheme_ref_len, "text_cer_grapheme": gr_cer, "confidence": crop.get("confidence"), "crop_path": crop.get("crop_path") or "", "crop_sha256": crop.get("crop_sha256") or "", "crop_xyxy": crop.get("crop_xyxy") or "", "detected_bbox": crop.get("detected_bbox") or "", "detected_area": crop.get("detected_area"), "crop_width_px": crop_width, "crop_height_px": crop_height, "crop_aspect_ratio": _ratio(crop_width, crop_height), "detected_width_px": detected_width, "detected_height_px": detected_height, "detected_aspect_ratio": _ratio(detected_width, detected_height), "reference_width_px": reference_width, "reference_height_px": reference_height, "reference_x_px": reference_x, "reference_y_px": reference_y, "reference_center_x_px": reference_center_x, "reference_center_y_px": reference_center_y, "reference_aspect_ratio": _ratio(reference_width, reference_height), "predicted_width_px": predicted_width, "predicted_height_px": predicted_height, "predicted_x_px": predicted_x, "predicted_y_px": predicted_y, "predicted_center_x_px": predicted_center_x, "predicted_center_y_px": predicted_center_y, "predicted_aspect_ratio": _ratio(predicted_width, predicted_height), "crop_px_per_codepoint": _ratio(crop_width, float(codepoint_ref_len)), "crop_px_per_grapheme": _ratio(crop_width, float(grapheme_ref_len)), } rows.append(joined) return rows def _predicted_overlap_candidates(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: candidates: list[dict[str, Any]] = [] for row in rows: if row.get("kind") not in {"matched", "prediction_unmatched"}: continue if not row.get("predicted_bbox"): continue candidates.append(row) return candidates def _annotate_unmatched_reference_prediction_overlaps(rows: list[dict[str, Any]]) -> None: candidates = _predicted_overlap_candidates(rows) for row in rows: if row.get("kind") != "reference_unmatched": continue best: dict[str, Any] | None = None best_iou = 0.0 for candidate in candidates: if candidate.get("sample_id") != row.get("sample_id") or candidate.get("page_index") != row.get("page_index"): continue iou = _bbox_iou(row.get("reference_bbox"), candidate.get("predicted_bbox")) if iou > best_iou: best_iou = iou best = candidate row["best_any_prediction_iou"] = best_iou row["best_any_prediction_line_id"] = best.get("predicted_line_id") if best is not None else "" row["best_any_prediction_kind"] = best.get("kind") if best is not None else "" row["best_any_prediction_bbox"] = best.get("predicted_bbox") if best is not None else "" row["best_any_prediction_text"] = best.get("predicted_text") if best is not None else "" row["best_any_prediction_confidence"] = best.get("confidence") if best is not None else None def _bucket_counts(rows: list[dict[str, Any]]) -> dict[str, int]: counts: dict[str, int] = {} for row in rows: bucket = str(row.get("bucket") or "unknown") counts[bucket] = counts.get(bucket, 0) + 1 return dict(sorted(counts.items())) def _bucket_stats(rows: list[dict[str, Any]]) -> dict[str, dict[str, float | int | None]]: buckets = sorted({str(row.get("bucket") or "unknown") for row in rows}) stats: dict[str, dict[str, float | int | None]] = {} for bucket in buckets: bucket_rows = [row for row in rows if row.get("bucket") == bucket] stats[bucket] = { "count": len(bucket_rows), "mean_iou": _mean(bucket_rows, "iou"), "mean_confidence": _mean(bucket_rows, "confidence"), "mean_codepoint_ref_len": _mean(bucket_rows, "text_codepoint_ref_len"), "mean_grapheme_ref_len": _mean(bucket_rows, "text_grapheme_ref_len"), "mean_codepoint_cer": _mean(bucket_rows, "text_cer_codepoint"), "mean_grapheme_cer": _mean(bucket_rows, "text_cer_grapheme"), "mean_crop_width_px": _mean(bucket_rows, "crop_width_px"), "mean_crop_height_px": _mean(bucket_rows, "crop_height_px"), "mean_crop_aspect_ratio": _mean(bucket_rows, "crop_aspect_ratio"), "mean_reference_x_px": _mean(bucket_rows, "reference_x_px"), "mean_reference_y_px": _mean(bucket_rows, "reference_y_px"), "mean_reference_width_px": _mean(bucket_rows, "reference_width_px"), "mean_reference_height_px": _mean(bucket_rows, "reference_height_px"), "mean_reference_aspect_ratio": _mean(bucket_rows, "reference_aspect_ratio"), "mean_predicted_x_px": _mean(bucket_rows, "predicted_x_px"), "mean_predicted_y_px": _mean(bucket_rows, "predicted_y_px"), "mean_predicted_width_px": _mean(bucket_rows, "predicted_width_px"), "mean_predicted_height_px": _mean(bucket_rows, "predicted_height_px"), "mean_predicted_aspect_ratio": _mean(bucket_rows, "predicted_aspect_ratio"), "mean_crop_px_per_codepoint": _mean(bucket_rows, "crop_px_per_codepoint"), "mean_crop_px_per_grapheme": _mean(bucket_rows, "crop_px_per_grapheme"), } return stats def _mean(rows: list[dict[str, Any]], key: str) -> float | None: values = [_float(row, key) for row in rows if row.get(key) not in ("", None)] return sum(values) / len(values) if values else None def _crop_width_band(width: Any) -> str: if width in ("", None): return "missing" value = _float({"width": width}, "width", -1.0) if value < 0: return "missing" if value <= 256: return "000-256" if value <= 384: return "257-384" if value <= 512: return "385-512" if value <= 768: return "513-768" return "769-plus" def _crop_width_band_stats(rows: list[dict[str, Any]]) -> dict[str, dict[str, float | int | None]]: matched = [row for row in rows if row.get("kind") == "matched"] bands = sorted({_crop_width_band(row.get("crop_width_px")) for row in matched}) stats: dict[str, dict[str, float | int | None]] = {} for band in bands: band_rows = [row for row in matched if _crop_width_band(row.get("crop_width_px")) == band] stats[band] = { "count": len(band_rows), "mean_iou": _mean(band_rows, "iou"), "mean_confidence": _mean(band_rows, "confidence"), "mean_codepoint_ref_len": _mean(band_rows, "text_codepoint_ref_len"), "mean_codepoint_cer": _mean(band_rows, "text_cer_codepoint"), "mean_grapheme_cer": _mean(band_rows, "text_cer_grapheme"), "mean_crop_aspect_ratio": _mean(band_rows, "crop_aspect_ratio"), } return stats def _text_codepoint_len(value: Any) -> int: if value in ("", None): return 0 return len(str(value)) def _codepoint_error_mass(rows: list[dict[str, Any]]) -> dict[str, Any]: sources = { "matched_good_geometry_bad_text": 0, "matched_good_geometry_acceptable_text": 0, "matched_low_geometry_bad_text": 0, "matched_low_geometry_acceptable_text": 0, "missed_reference_text": 0, "extra_prediction_text": 0, "other": 0, } for row in rows: kind = row.get("kind") if kind == "matched": bucket = str(row.get("bucket") or "other") sources[bucket if bucket in sources else "other"] += _int(row, "text_codepoint_edits") elif kind == "reference_unmatched": ref_len = _int(row, "text_codepoint_ref_len") sources["missed_reference_text"] += ref_len if ref_len > 0 else _text_codepoint_len(row.get("reference_text")) elif kind == "prediction_unmatched": sources["extra_prediction_text"] += _text_codepoint_len(row.get("predicted_text")) else: sources["other"] += _int(row, "text_codepoint_edits") total = sum(sources.values()) fractions = {key: value / total if total else 0.0 for key, value in sources.items()} dominant_source = max(sources, key=lambda key: (sources[key], key)) if total else None return { "total": total, "by_source": sources, "fractions": fractions, "dominant_source": dominant_source, } def _spatial_group_key(row: dict[str, Any]) -> tuple[str, str]: return str(row.get("sample_id") or ""), str(row.get("page_index") or "") def _reference_spatial_context(rows: list[dict[str, Any]]) -> dict[tuple[str, str], dict[str, float]]: context: dict[tuple[str, str], dict[str, float]] = {} for row in rows: if row.get("kind") not in {"matched", "reference_unmatched"}: continue center_x = row.get("reference_center_x_px") center_y = row.get("reference_center_y_px") if center_x in ("", None) or center_y in ("", None): continue x = float(center_x) y = float(center_y) key = _spatial_group_key(row) if key not in context: context[key] = {"min_x": x, "max_x": x, "min_y": y, "max_y": y} continue item = context[key] item["min_x"] = min(item["min_x"], x) item["max_x"] = max(item["max_x"], x) item["min_y"] = min(item["min_y"], y) item["max_y"] = max(item["max_y"], y) return context def _axis_band(value: Any, *, minimum: float, maximum: float, labels: tuple[str, str, str]) -> str: if value in ("", None): return "unknown" span = maximum - minimum if span <= 0: return labels[1] ratio = (float(value) - minimum) / span if ratio < 1 / 3: return labels[0] if ratio < 2 / 3: return labels[1] return labels[2] def _empty_spatial_bucket() -> dict[str, float | int | None]: return { "reference_count": 0, "unmatched_reference_count": 0, "unmatched_reference_fraction": 0.0, "codepoint_error_mass": 0, "mean_reference_x_px": None, "mean_reference_y_px": None, "mean_reference_width_px": None, "mean_reference_height_px": None, } def _finalize_spatial_bucket(rows: list[dict[str, Any]], unmatched_rows: list[dict[str, Any]]) -> dict[str, float | int | None]: reference_count = len(rows) unmatched_count = len(unmatched_rows) return { "reference_count": reference_count, "unmatched_reference_count": unmatched_count, "unmatched_reference_fraction": unmatched_count / reference_count if reference_count else 0.0, "codepoint_error_mass": sum( _int(row, "text_codepoint_ref_len") if _int(row, "text_codepoint_ref_len") > 0 else _text_codepoint_len(row.get("reference_text")) for row in unmatched_rows ), "mean_reference_x_px": _mean(unmatched_rows, "reference_x_px"), "mean_reference_y_px": _mean(unmatched_rows, "reference_y_px"), "mean_reference_width_px": _mean(unmatched_rows, "reference_width_px"), "mean_reference_height_px": _mean(unmatched_rows, "reference_height_px"), } def _reference_spatial_stats(rows: list[dict[str, Any]]) -> dict[str, Any]: context = _reference_spatial_context(rows) reference_rows: list[dict[str, Any]] = [] for row in rows: if row.get("kind") not in {"matched", "reference_unmatched"}: continue group = context.get(_spatial_group_key(row)) if group is None: continue y_band = _axis_band( row.get("reference_center_y_px"), minimum=group["min_y"], maximum=group["max_y"], labels=("top", "middle", "bottom"), ) x_band = _axis_band( row.get("reference_center_x_px"), minimum=group["min_x"], maximum=group["max_x"], labels=("left", "center", "right"), ) enriched = dict(row) enriched["reference_y_band"] = y_band enriched["reference_x_band"] = x_band enriched["reference_xy_band"] = f"{y_band}/{x_band}" reference_rows.append(enriched) def grouped(field: str, ordered_keys: tuple[str, ...] | None = None) -> dict[str, dict[str, float | int | None]]: keys = list(ordered_keys or tuple(sorted({str(row[field]) for row in reference_rows}))) stats: dict[str, dict[str, float | int | None]] = {} for key in keys: bucket_rows = [row for row in reference_rows if row[field] == key] unmatched_rows = [row for row in bucket_rows if row.get("kind") == "reference_unmatched"] stats[key] = _finalize_spatial_bucket(bucket_rows, unmatched_rows) if bucket_rows else _empty_spatial_bucket() return stats xy_keys = tuple( f"{y}/{x}" for y in ("top", "middle", "bottom") for x in ("left", "center", "right") ) by_y_band = grouped("reference_y_band", ("top", "middle", "bottom", "unknown")) by_x_band = grouped("reference_x_band", ("left", "center", "right", "unknown")) by_xy_band = grouped("reference_xy_band", (*xy_keys, "unknown/unknown")) top_unmatched_bands = sorted( ( {"band": band, **stats} for band, stats in by_xy_band.items() if int(stats["unmatched_reference_count"] or 0) > 0 ), key=lambda row: ( int(row["codepoint_error_mass"] or 0), int(row["unmatched_reference_count"] or 0), float(row["unmatched_reference_fraction"] or 0.0), str(row["band"]), ), reverse=True, ) return { "reference_count": len(reference_rows), "unmatched_reference_count": sum(row.get("kind") == "reference_unmatched" for row in reference_rows), "by_y_band": by_y_band, "by_x_band": by_x_band, "by_xy_band": by_xy_band, "top_unmatched_bands": top_unmatched_bands, } def _unmatched_reference_overlap_stats(rows: list[dict[str, Any]]) -> dict[str, Any]: unmatched = [row for row in rows if row.get("kind") == "reference_unmatched"] source_counts: dict[str, int] = {} for row in unmatched: source = str(row.get("best_any_prediction_kind") or "none") source_counts[source] = source_counts.get(source, 0) + 1 return { "reference_unmatched_count": len(unmatched), "best_any_prediction_iou_ge_010": sum(_float(row, "best_any_prediction_iou") >= 0.10 for row in unmatched), "best_any_prediction_iou_ge_030": sum(_float(row, "best_any_prediction_iou") >= 0.30 for row in unmatched), "best_any_prediction_iou_ge_050": sum(_float(row, "best_any_prediction_iou") >= 0.50 for row in unmatched), "best_any_prediction_iou_ge_085": sum(_float(row, "best_any_prediction_iou") >= 0.85 for row in unmatched), "best_any_prediction_source_counts": dict(sorted(source_counts.items())), "mean_best_any_prediction_iou": _mean(unmatched, "best_any_prediction_iou"), } def _top_failures(rows: list[dict[str, Any]], limit: int) -> list[dict[str, Any]]: candidates = [row for row in rows if row["kind"] == "matched"] candidates.sort( key=lambda row: ( _int(row, "text_codepoint_edits"), _float(row, "text_cer_grapheme"), _float(row, "text_cer_codepoint"), ), reverse=True, ) return candidates[:limit] def _top_unmatched_references(rows: list[dict[str, Any]], limit: int) -> list[dict[str, Any]]: candidates = [row for row in rows if row["kind"] == "reference_unmatched"] candidates.sort( key=lambda row: ( _int(row, "text_codepoint_ref_len"), _float(row, "best_any_prediction_iou"), _float(row, "reference_width_px") * _float(row, "reference_height_px"), _float(row, "reference_y_px"), _float(row, "reference_x_px"), ), reverse=True, ) return candidates[:limit] def _top_unmatched_predictions(rows: list[dict[str, Any]], limit: int) -> list[dict[str, Any]]: candidates = [row for row in rows if row["kind"] == "prediction_unmatched"] candidates.sort( key=lambda row: ( _float(row, "confidence", -1.0), len(str(row.get("predicted_text") or "")), _float(row, "predicted_width_px") * _float(row, "predicted_height_px"), ), reverse=True, ) return candidates[:limit] def _near_miss_unmatched_pairs(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: references = [row for row in rows if row["kind"] == "reference_unmatched"] predictions = [row for row in rows if row["kind"] == "prediction_unmatched"] pairs: list[dict[str, Any]] = [] for reference in references: sample_id = reference.get("sample_id") page_index = reference.get("page_index") candidates = [ prediction for prediction in predictions if prediction.get("sample_id") == sample_id and prediction.get("page_index") == page_index ] best_prediction: dict[str, Any] | None = None best_iou = 0.0 for prediction in candidates: iou = _bbox_iou(reference.get("reference_bbox"), prediction.get("predicted_bbox")) if iou > best_iou: best_iou = iou best_prediction = prediction if best_prediction is None: continue pairs.append( { "sample_id": sample_id, "page_index": page_index, "reference_line_id": reference.get("reference_line_id") or "", "predicted_line_id": best_prediction.get("predicted_line_id") or "", "iou": best_iou, "reference_bbox": reference.get("reference_bbox") or "", "predicted_bbox": best_prediction.get("predicted_bbox") or "", "reference_text": reference.get("reference_text") or "", "predicted_text": best_prediction.get("predicted_text") or "", "confidence": best_prediction.get("confidence"), "reference_width_px": reference.get("reference_width_px"), "reference_height_px": reference.get("reference_height_px"), "predicted_width_px": best_prediction.get("predicted_width_px"), "predicted_height_px": best_prediction.get("predicted_height_px"), "crop_path": best_prediction.get("crop_path") or "", "crop_sha256": best_prediction.get("crop_sha256") or "", } ) pairs.sort(key=lambda row: _float(row, "iou"), reverse=True) return pairs def _write_rows(path: Path, rows: list[dict[str, Any]]) -> None: fields = [ "sample_id", "kind", "bucket", "page_index", "reference_line_id", "predicted_line_id", "iou", "text_codepoint_edits", "text_codepoint_ref_len", "text_cer_codepoint", "text_grapheme_edits", "text_grapheme_ref_len", "text_cer_grapheme", "confidence", "crop_path", "crop_sha256", "reference_bbox", "predicted_bbox", "crop_xyxy", "detected_bbox", "detected_area", "crop_width_px", "crop_height_px", "crop_aspect_ratio", "detected_width_px", "detected_height_px", "detected_aspect_ratio", "reference_width_px", "reference_height_px", "reference_x_px", "reference_y_px", "reference_center_x_px", "reference_center_y_px", "reference_aspect_ratio", "predicted_width_px", "predicted_height_px", "predicted_x_px", "predicted_y_px", "predicted_center_x_px", "predicted_center_y_px", "predicted_aspect_ratio", "best_any_prediction_iou", "best_any_prediction_line_id", "best_any_prediction_kind", "best_any_prediction_bbox", "best_any_prediction_confidence", "crop_px_per_codepoint", "crop_px_per_grapheme", "reference_text", "predicted_text", "best_any_prediction_text", ] with path.open("w", encoding="utf-8", newline="") as handle: writer = csv.DictWriter(handle, fieldnames=fields) writer.writeheader() for row in rows: payload = dict(row) for key in ("crop_xyxy", "detected_bbox"): if isinstance(payload.get(key), list): payload[key] = json.dumps(payload[key], ensure_ascii=False) writer.writerow({field: payload.get(field, "") for field in fields}) def _write_near_miss_pairs(path: Path, rows: list[dict[str, Any]]) -> None: fields = [ "sample_id", "page_index", "reference_line_id", "predicted_line_id", "iou", "confidence", "reference_bbox", "predicted_bbox", "reference_width_px", "reference_height_px", "best_any_prediction_iou", "best_any_prediction_line_id", "best_any_prediction_kind", "best_any_prediction_bbox", "best_any_prediction_confidence", "predicted_width_px", "predicted_height_px", "crop_path", "crop_sha256", "reference_text", "predicted_text", ] with path.open("w", encoding="utf-8", newline="") as handle: writer = csv.DictWriter(handle, fieldnames=fields) writer.writeheader() for row in rows: writer.writerow({field: row.get(field, "") for field in fields}) def _write_report(path: Path, result: dict[str, Any]) -> None: summary = result["summary"] buckets = summary["bucket_counts"] lines = [ "# Page Line Failure Analysis", "", "This report joins page line-alignment rows with crop provenance. It is diagnostic evidence, not a claim-safe score.", "", f"- line alignment: `{result['line_alignment_csv']}`", f"- crop manifest: `{result.get('crop_manifest') or ''}`", f"- rows: {summary['row_count']}", f"- matched rows: {summary['matched_count']}", f"- unmatched references: {summary['reference_unmatched_count']}", f"- unmatched predictions: {summary['prediction_unmatched_count']}", f"- high-IoU threshold: {result['thresholds']['high_iou']}", f"- bad codepoint CER threshold: {result['thresholds']['bad_codepoint_cer']}", f"- bad grapheme CER threshold: {result['thresholds']['bad_grapheme_cer']}", f"- high-IoU bad-text matched rows: {buckets.get('matched_good_geometry_bad_text', 0)}", f"- high-IoU acceptable-text matched rows: {buckets.get('matched_good_geometry_acceptable_text', 0)}", f"- mean matched IoU: {summary['matched_mean_iou']:.6f}", f"- mean matched codepoint CER: {summary['matched_mean_codepoint_cer']:.6f}", f"- mean matched grapheme CER: {summary['matched_mean_grapheme_cer']:.6f}", f"- mean unmatched-reference y: {_format_optional_float(summary['bucket_stats'].get('reference_unmatched', {}).get('mean_reference_y_px'))}", f"- mean unmatched-prediction confidence: {_format_optional_float(summary['bucket_stats'].get('prediction_unmatched', {}).get('mean_confidence'))}", f"- unmatched reference/prediction near-miss pairs at IoU >= 0.30: {summary['near_miss_unmatched_pair_count_iou_ge_030']}", f"- unmatched reference/prediction near-miss pairs at IoU >= 0.50: {summary['near_miss_unmatched_pair_count_iou_ge_050']}", f"- unmatched references with any prediction IoU >= 0.30: {summary['unmatched_reference_overlap_stats']['best_any_prediction_iou_ge_030']}", f"- unmatched references with any prediction IoU >= 0.50: {summary['unmatched_reference_overlap_stats']['best_any_prediction_iou_ge_050']}", f"- unmatched references with no overlapping prediction: {summary['unmatched_reference_overlap_stats']['best_any_prediction_source_counts'].get('none', 0)}", f"- dominant codepoint error source: {summary['codepoint_error_mass']['dominant_source'] or 'n/a'}", f"- total attributed codepoint error mass: {summary['codepoint_error_mass']['total']}", "", "## Geometry Signal", "", f"- mean high-IoU bad-text crop width: {_format_optional_float(summary['bucket_stats'].get('matched_good_geometry_bad_text', {}).get('mean_crop_width_px'))}", f"- mean high-IoU acceptable-text crop width: {_format_optional_float(summary['bucket_stats'].get('matched_good_geometry_acceptable_text', {}).get('mean_crop_width_px'))}", f"- mean high-IoU bad-text crop aspect ratio: {_format_optional_float(summary['bucket_stats'].get('matched_good_geometry_bad_text', {}).get('mean_crop_aspect_ratio'))}", f"- mean high-IoU acceptable-text crop aspect ratio: {_format_optional_float(summary['bucket_stats'].get('matched_good_geometry_acceptable_text', {}).get('mean_crop_aspect_ratio'))}", f"- mean unmatched-reference width: {_format_optional_float(summary['bucket_stats'].get('reference_unmatched', {}).get('mean_reference_width_px'))}", f"- mean unmatched-reference height: {_format_optional_float(summary['bucket_stats'].get('reference_unmatched', {}).get('mean_reference_height_px'))}", f"- mean unmatched-prediction width: {_format_optional_float(summary['bucket_stats'].get('prediction_unmatched', {}).get('mean_predicted_width_px'))}", f"- mean unmatched-prediction height: {_format_optional_float(summary['bucket_stats'].get('prediction_unmatched', {}).get('mean_predicted_height_px'))}", "", "## Missed Reference Spatial Bands", "", ] spatial = summary["unmatched_reference_spatial_stats"] for band, stats in spatial["by_y_band"].items(): lines.append( "- " f"{band}: {stats['unmatched_reference_count']}/{stats['reference_count']} unmatched " f"({float(stats['unmatched_reference_fraction']):.3f}), " f"error mass {stats['codepoint_error_mass']}" ) lines.extend(["", "By horizontal band:", ""]) for band, stats in spatial["by_x_band"].items(): lines.append( "- " f"{band}: {stats['unmatched_reference_count']}/{stats['reference_count']} unmatched " f"({float(stats['unmatched_reference_fraction']):.3f}), " f"error mass {stats['codepoint_error_mass']}" ) lines.extend(["", "Top combined bands:", ""]) for row in spatial["top_unmatched_bands"][:5]: lines.append( "- " f"{row['band']}: {row['unmatched_reference_count']}/{row['reference_count']} unmatched " f"({float(row['unmatched_reference_fraction']):.3f}), " f"error mass {row['codepoint_error_mass']}" ) lines.extend([ "", "## Error Mass", "", ]) for source, count in summary["codepoint_error_mass"]["by_source"].items(): fraction = summary["codepoint_error_mass"]["fractions"].get(source, 0.0) lines.append(f"- {source}: {count} ({fraction:.3f})") lines.extend([ "", "## Top Failures", "", ]) for row in result["top_failures"][:10]: lines.append( "- " f"{row['reference_line_id']} -> {row['predicted_line_id']}: " f"IoU {row['iou']:.3f}, cp CER {row['text_cer_codepoint']:.3f}, " f"gr CER {row['text_cer_grapheme']:.3f}, crop `{row['crop_path']}`" ) lines.extend(["", "## Top Unmatched References", ""]) for row in result.get("top_unmatched_references", [])[:10]: lines.append( "- " f"{row['reference_line_id']}: ref len {row['text_codepoint_ref_len']}, " f"bbox `{row['reference_bbox']}`, " f"best any prediction IoU {_format_optional_float(row.get('best_any_prediction_iou'))} " f"({row.get('best_any_prediction_kind') or 'none'} {row.get('best_any_prediction_line_id') or ''})" ) lines.extend(["", "## Top Unmatched Predictions", ""]) for row in result.get("top_unmatched_predictions", [])[:10]: lines.append( "- " f"{row['predicted_line_id']}: confidence {_format_optional_float(row.get('confidence'))}, " f"bbox `{row['predicted_bbox']}`, text `{row['predicted_text']}`" ) lines.extend(["", "## Top Unmatched Near Misses", ""]) for row in result.get("near_miss_unmatched_pairs", [])[:10]: lines.append( "- " f"{row['reference_line_id']} -> {row['predicted_line_id']}: " f"IoU {row['iou']:.3f}, confidence {_format_optional_float(row.get('confidence'))}, " f"crop `{row['crop_path']}`" ) path.write_text("\n".join(lines) + "\n", encoding="utf-8") def _format_optional_float(value: float | int | None) -> str: if value is None: return "n/a" return f"{float(value):.6f}" def analyze(args: argparse.Namespace) -> dict[str, Any]: if args.high_iou_threshold <= 0 or args.high_iou_threshold > 1: raise ValueError("--high-iou-threshold must be in (0, 1]") if args.bad_codepoint_cer < 0 or args.bad_grapheme_cer < 0: raise ValueError("bad CER thresholds must be non-negative") args.out.mkdir(parents=True, exist_ok=True) alignment_rows = _read_csv(args.line_alignment_csv) crop_rows = _read_jsonl(args.crop_manifest) joined = _joined_rows( alignment_rows, crop_rows, high_iou_threshold=args.high_iou_threshold, bad_codepoint_cer=args.bad_codepoint_cer, bad_grapheme_cer=args.bad_grapheme_cer, ) _annotate_unmatched_reference_prediction_overlaps(joined) matched = [row for row in joined if row["kind"] == "matched"] top = _top_failures(joined, args.top_k) top_unmatched_references = _top_unmatched_references(joined, args.top_k) top_unmatched_predictions = _top_unmatched_predictions(joined, args.top_k) near_miss_unmatched_pairs = _near_miss_unmatched_pairs(joined) bucket_counts = _bucket_counts(joined) bucket_stats = _bucket_stats(joined) crop_width_band_stats = _crop_width_band_stats(joined) codepoint_error_mass = _codepoint_error_mass(joined) unmatched_reference_spatial_stats = _reference_spatial_stats(joined) unmatched_reference_overlap_stats = _unmatched_reference_overlap_stats(joined) result = { "created_at": datetime.now(timezone.utc).isoformat(), "machine": platform.node(), "tool": "tools/analyze_page_line_failures.py", "line_alignment_csv": str(args.line_alignment_csv), "line_alignment_csv_sha256": sha256_file(args.line_alignment_csv), "crop_manifest": str(args.crop_manifest) if args.crop_manifest else None, "crop_manifest_sha256": sha256_file(args.crop_manifest) if args.crop_manifest else None, "thresholds": { "high_iou": args.high_iou_threshold, "bad_codepoint_cer": args.bad_codepoint_cer, "bad_grapheme_cer": args.bad_grapheme_cer, }, "summary": { "row_count": len(joined), "matched_count": len(matched), "reference_unmatched_count": sum(row["kind"] == "reference_unmatched" for row in joined), "prediction_unmatched_count": sum(row["kind"] == "prediction_unmatched" for row in joined), "bucket_counts": bucket_counts, "bucket_stats": bucket_stats, "crop_width_band_stats": crop_width_band_stats, "codepoint_error_mass": codepoint_error_mass, "unmatched_reference_spatial_stats": unmatched_reference_spatial_stats, "unmatched_reference_overlap_stats": unmatched_reference_overlap_stats, "matched_mean_iou": _mean(matched, "iou") or 0.0, "matched_mean_codepoint_cer": _mean(matched, "text_cer_codepoint") or 0.0, "matched_mean_grapheme_cer": _mean(matched, "text_cer_grapheme") or 0.0, "matched_good_geometry_bad_text_fraction": bucket_counts.get("matched_good_geometry_bad_text", 0) / max(len(matched), 1), "near_miss_unmatched_pair_count_iou_ge_030": sum( _float(row, "iou") >= 0.30 for row in near_miss_unmatched_pairs ), "near_miss_unmatched_pair_count_iou_ge_050": sum( _float(row, "iou") >= 0.50 for row in near_miss_unmatched_pairs ), }, "top_failures": top, "top_unmatched_references": top_unmatched_references, "top_unmatched_predictions": top_unmatched_predictions, "near_miss_unmatched_pairs": near_miss_unmatched_pairs[: args.top_k], } rows_path = args.out / "line-failures.csv" top_path = args.out / "top-line-failures.csv" top_unmatched_references_path = args.out / "top-unmatched-references.csv" top_unmatched_predictions_path = args.out / "top-unmatched-predictions.csv" near_miss_unmatched_pairs_path = args.out / "near-miss-unmatched-pairs.csv" result_path = args.out / "line-failure-analysis.json" report_path = args.out / "line-failure-analysis.md" _write_rows(rows_path, joined) _write_rows(top_path, top) _write_rows(top_unmatched_references_path, top_unmatched_references) _write_rows(top_unmatched_predictions_path, top_unmatched_predictions) _write_near_miss_pairs(near_miss_unmatched_pairs_path, near_miss_unmatched_pairs) result["rows_csv"] = str(rows_path) result["top_failures_csv"] = str(top_path) result["top_unmatched_references_csv"] = str(top_unmatched_references_path) result["top_unmatched_predictions_csv"] = str(top_unmatched_predictions_path) result["near_miss_unmatched_pairs_csv"] = str(near_miss_unmatched_pairs_path) result["result_path"] = str(result_path) result["report_path"] = str(report_path) result_path.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") _write_report(report_path, result) return result def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--line-alignment-csv", type=Path, required=True) parser.add_argument("--crop-manifest", type=Path) parser.add_argument("--out", type=Path, required=True) parser.add_argument("--high-iou-threshold", type=float, default=0.85) parser.add_argument("--bad-codepoint-cer", type=float, default=0.30) parser.add_argument("--bad-grapheme-cer", type=float, default=0.50) parser.add_argument("--top-k", type=int, default=50) args = parser.parse_args(argv) result = analyze(args) summary = result["summary"] print( f"wrote {result['result_path']} matched={summary['matched_count']} " f"good_geometry_bad_text={summary['bucket_counts'].get('matched_good_geometry_bad_text', 0)} " f"dominant_error_source={summary['codepoint_error_mass']['dominant_source'] or 'n/a'}" ) return 0 if __name__ == "__main__": raise SystemExit(main())