Add 1000 files (ocr-tech/tests/test_audit_limbu_book_missed_line_review_coverage.py ...)
d82db76 verified Download ocr-tech/tools/analyze_page_line_failures.py from prosperious/ocr-projects-archive: direct link, hf CLI and curl.
- Browser
- Download file 43.5 kB
-
https://huggingface.co/prosperious/ocr-projects-archive/resolve/main/ocr-tech/tools/analyze_page_line_failures.py
- Command line
-
hf download hf://prosperious/ocr-projects-archive/ocr-tech/tools/analyze_page_line_failures.py
-
curl -L -o analyze_page_line_failures.py https://huggingface.co/prosperious/ocr-projects-archive/resolve/main/ocr-tech/tools/analyze_page_line_failures.py
43.5 kB
| #!/usr/bin/env python3 | |
| """Analyze page OCR line-alignment failures with crop provenance. | |
| The page scorer already emits `line-alignment.csv`. This diagnostic joins those | |
| alignment rows to an image-line crop manifest when available, then writes ranked | |
| failure examples and aggregate buckets that distinguish geometry failures from | |
| recognizer/crop-quality failures on well-aligned boxes. | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import csv | |
| import json | |
| import platform | |
| import ast | |
| from datetime import datetime, timezone | |
| from pathlib import Path | |
| from typing import Any | |
| from ocrtech.manifest import sha256_file | |
| def _read_csv(path: Path) -> list[dict[str, str]]: | |
| if not path.exists(): | |
| raise FileNotFoundError(f"missing CSV file: {path}") | |
| with path.open("r", encoding="utf-8", newline="") as handle: | |
| rows = list(csv.DictReader(handle)) | |
| if not rows: | |
| raise ValueError(f"CSV file contains no rows: {path}") | |
| return rows | |
| def _read_jsonl(path: Path | None) -> dict[tuple[str, str], dict[str, Any]]: | |
| if path is None: | |
| return {} | |
| if not path.exists(): | |
| raise FileNotFoundError(f"missing JSONL file: {path}") | |
| rows: dict[tuple[str, str], dict[str, Any]] = {} | |
| with path.open("r", encoding="utf-8") as handle: | |
| for line_number, line in enumerate(handle, start=1): | |
| if not line.strip(): | |
| continue | |
| payload = json.loads(line) | |
| if not isinstance(payload, dict): | |
| raise ValueError(f"{path}:{line_number}: JSONL row must be an object") | |
| sample_id = str(payload.get("sample_id") or "") | |
| line_id = str(payload.get("line_id") or payload.get("predicted_line_id") or "") | |
| if line_id: | |
| rows[(sample_id, line_id)] = payload | |
| return rows | |
| def _float(row: dict[str, Any], key: str, default: float = 0.0) -> float: | |
| value = row.get(key, default) | |
| try: | |
| return float(value) | |
| except (TypeError, ValueError): | |
| return default | |
| def _int(row: dict[str, Any], key: str, default: int = 0) -> int: | |
| value = row.get(key, default) | |
| try: | |
| return int(value) | |
| except (TypeError, ValueError): | |
| return default | |
| def _number_sequence(value: Any) -> list[float]: | |
| if value in ("", None): | |
| return [] | |
| if isinstance(value, (list, tuple)): | |
| raw_items = value | |
| elif isinstance(value, str): | |
| try: | |
| parsed = json.loads(value) | |
| except json.JSONDecodeError: | |
| try: | |
| parsed = ast.literal_eval(value) | |
| except (SyntaxError, ValueError): | |
| return [] | |
| if not isinstance(parsed, (list, tuple)): | |
| return [] | |
| raw_items = parsed | |
| else: | |
| return [] | |
| values: list[float] = [] | |
| for item in raw_items: | |
| try: | |
| values.append(float(item)) | |
| except (TypeError, ValueError): | |
| return [] | |
| return values | |
| def _bbox_geometry(value: Any) -> tuple[float | None, float | None, float | None, float | None]: | |
| values = _number_sequence(value) | |
| if len(values) != 4: | |
| return None, None, None, None | |
| x, y, width, height = values | |
| if width < 0 or height < 0: | |
| return None, None, None, None | |
| return x, y, width, height | |
| def _bbox_width_height(value: Any) -> tuple[float | None, float | None]: | |
| _x, _y, width, height = _bbox_geometry(value) | |
| return width, height | |
| def _bbox_center(value: Any) -> tuple[float | None, float | None]: | |
| x, y, width, height = _bbox_geometry(value) | |
| if x is None or y is None or width is None or height is None: | |
| return None, None | |
| return x + width / 2.0, y + height / 2.0 | |
| def _xyxy_width_height(value: Any) -> tuple[float | None, float | None]: | |
| values = _number_sequence(value) | |
| if len(values) != 4: | |
| return None, None | |
| width = values[2] - values[0] | |
| height = values[3] - values[1] | |
| if width < 0 or height < 0: | |
| return None, None | |
| return width, height | |
| def _ratio(numerator: float | None, denominator: float | None) -> float | None: | |
| if numerator is None or denominator is None or denominator == 0: | |
| return None | |
| return numerator / denominator | |
| def _bbox_iou(left_bbox: Any, right_bbox: Any) -> float: | |
| left_x, left_y, left_w, left_h = _bbox_geometry(left_bbox) | |
| right_x, right_y, right_w, right_h = _bbox_geometry(right_bbox) | |
| if ( | |
| left_x is None | |
| or left_y is None | |
| or left_w is None | |
| or left_h is None | |
| or right_x is None | |
| or right_y is None | |
| or right_w is None | |
| or right_h is None | |
| ): | |
| return 0.0 | |
| left_x2 = left_x + left_w | |
| left_y2 = left_y + left_h | |
| right_x2 = right_x + right_w | |
| right_y2 = right_y + right_h | |
| inter_w = max(0.0, min(left_x2, right_x2) - max(left_x, right_x)) | |
| inter_h = max(0.0, min(left_y2, right_y2) - max(left_y, right_y)) | |
| inter_area = inter_w * inter_h | |
| if inter_area <= 0: | |
| return 0.0 | |
| union_area = left_w * left_h + right_w * right_h - inter_area | |
| return inter_area / union_area if union_area > 0 else 0.0 | |
| def _joined_rows( | |
| alignment_rows: list[dict[str, str]], | |
| crop_rows: dict[tuple[str, str], dict[str, Any]], | |
| *, | |
| high_iou_threshold: float, | |
| bad_codepoint_cer: float, | |
| bad_grapheme_cer: float, | |
| ) -> list[dict[str, Any]]: | |
| rows: list[dict[str, Any]] = [] | |
| for row in alignment_rows: | |
| sample_id = row.get("sample_id") or "" | |
| predicted_line_id = row.get("predicted_line_id") or "" | |
| crop = crop_rows.get((sample_id, predicted_line_id), crop_rows.get(("", predicted_line_id), {})) | |
| iou = _float(row, "iou") | |
| cp_cer = _float(row, "text_cer_codepoint") | |
| gr_cer = _float(row, "text_cer_grapheme") | |
| kind = row.get("kind") or "" | |
| crop_width, crop_height = _xyxy_width_height(crop.get("crop_xyxy")) | |
| detected_width, detected_height = _bbox_width_height(crop.get("detected_bbox")) | |
| reference_x, reference_y, reference_width, reference_height = _bbox_geometry(row.get("reference_bbox")) | |
| predicted_x, predicted_y, predicted_width, predicted_height = _bbox_geometry(row.get("predicted_bbox")) | |
| codepoint_ref_len = _int(row, "text_codepoint_ref_len") | |
| grapheme_ref_len = _int(row, "text_grapheme_ref_len") | |
| reference_center_x, reference_center_y = _bbox_center(row.get("reference_bbox")) | |
| predicted_center_x, predicted_center_y = _bbox_center(row.get("predicted_bbox")) | |
| bucket = kind | |
| if kind == "matched": | |
| good_geometry = iou >= high_iou_threshold | |
| bad_text = cp_cer >= bad_codepoint_cer or gr_cer >= bad_grapheme_cer | |
| if good_geometry and bad_text: | |
| bucket = "matched_good_geometry_bad_text" | |
| elif good_geometry: | |
| bucket = "matched_good_geometry_acceptable_text" | |
| elif bad_text: | |
| bucket = "matched_low_geometry_bad_text" | |
| else: | |
| bucket = "matched_low_geometry_acceptable_text" | |
| joined = { | |
| "sample_id": sample_id, | |
| "kind": kind, | |
| "bucket": bucket, | |
| "page_index": row.get("page_index") or "", | |
| "reference_line_id": row.get("reference_line_id") or "", | |
| "predicted_line_id": predicted_line_id, | |
| "iou": iou, | |
| "reference_bbox": row.get("reference_bbox") or "", | |
| "predicted_bbox": row.get("predicted_bbox") or "", | |
| "reference_text": row.get("reference_text") or "", | |
| "predicted_text": row.get("predicted_text") or "", | |
| "text_codepoint_edits": _int(row, "text_codepoint_edits"), | |
| "text_codepoint_ref_len": codepoint_ref_len, | |
| "text_cer_codepoint": cp_cer, | |
| "text_grapheme_edits": _int(row, "text_grapheme_edits"), | |
| "text_grapheme_ref_len": grapheme_ref_len, | |
| "text_cer_grapheme": gr_cer, | |
| "confidence": crop.get("confidence"), | |
| "crop_path": crop.get("crop_path") or "", | |
| "crop_sha256": crop.get("crop_sha256") or "", | |
| "crop_xyxy": crop.get("crop_xyxy") or "", | |
| "detected_bbox": crop.get("detected_bbox") or "", | |
| "detected_area": crop.get("detected_area"), | |
| "crop_width_px": crop_width, | |
| "crop_height_px": crop_height, | |
| "crop_aspect_ratio": _ratio(crop_width, crop_height), | |
| "detected_width_px": detected_width, | |
| "detected_height_px": detected_height, | |
| "detected_aspect_ratio": _ratio(detected_width, detected_height), | |
| "reference_width_px": reference_width, | |
| "reference_height_px": reference_height, | |
| "reference_x_px": reference_x, | |
| "reference_y_px": reference_y, | |
| "reference_center_x_px": reference_center_x, | |
| "reference_center_y_px": reference_center_y, | |
| "reference_aspect_ratio": _ratio(reference_width, reference_height), | |
| "predicted_width_px": predicted_width, | |
| "predicted_height_px": predicted_height, | |
| "predicted_x_px": predicted_x, | |
| "predicted_y_px": predicted_y, | |
| "predicted_center_x_px": predicted_center_x, | |
| "predicted_center_y_px": predicted_center_y, | |
| "predicted_aspect_ratio": _ratio(predicted_width, predicted_height), | |
| "crop_px_per_codepoint": _ratio(crop_width, float(codepoint_ref_len)), | |
| "crop_px_per_grapheme": _ratio(crop_width, float(grapheme_ref_len)), | |
| } | |
| rows.append(joined) | |
| return rows | |
| def _predicted_overlap_candidates(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: | |
| candidates: list[dict[str, Any]] = [] | |
| for row in rows: | |
| if row.get("kind") not in {"matched", "prediction_unmatched"}: | |
| continue | |
| if not row.get("predicted_bbox"): | |
| continue | |
| candidates.append(row) | |
| return candidates | |
| def _annotate_unmatched_reference_prediction_overlaps(rows: list[dict[str, Any]]) -> None: | |
| candidates = _predicted_overlap_candidates(rows) | |
| for row in rows: | |
| if row.get("kind") != "reference_unmatched": | |
| continue | |
| best: dict[str, Any] | None = None | |
| best_iou = 0.0 | |
| for candidate in candidates: | |
| if candidate.get("sample_id") != row.get("sample_id") or candidate.get("page_index") != row.get("page_index"): | |
| continue | |
| iou = _bbox_iou(row.get("reference_bbox"), candidate.get("predicted_bbox")) | |
| if iou > best_iou: | |
| best_iou = iou | |
| best = candidate | |
| row["best_any_prediction_iou"] = best_iou | |
| row["best_any_prediction_line_id"] = best.get("predicted_line_id") if best is not None else "" | |
| row["best_any_prediction_kind"] = best.get("kind") if best is not None else "" | |
| row["best_any_prediction_bbox"] = best.get("predicted_bbox") if best is not None else "" | |
| row["best_any_prediction_text"] = best.get("predicted_text") if best is not None else "" | |
| row["best_any_prediction_confidence"] = best.get("confidence") if best is not None else None | |
| def _bucket_counts(rows: list[dict[str, Any]]) -> dict[str, int]: | |
| counts: dict[str, int] = {} | |
| for row in rows: | |
| bucket = str(row.get("bucket") or "unknown") | |
| counts[bucket] = counts.get(bucket, 0) + 1 | |
| return dict(sorted(counts.items())) | |
| def _bucket_stats(rows: list[dict[str, Any]]) -> dict[str, dict[str, float | int | None]]: | |
| buckets = sorted({str(row.get("bucket") or "unknown") for row in rows}) | |
| stats: dict[str, dict[str, float | int | None]] = {} | |
| for bucket in buckets: | |
| bucket_rows = [row for row in rows if row.get("bucket") == bucket] | |
| stats[bucket] = { | |
| "count": len(bucket_rows), | |
| "mean_iou": _mean(bucket_rows, "iou"), | |
| "mean_confidence": _mean(bucket_rows, "confidence"), | |
| "mean_codepoint_ref_len": _mean(bucket_rows, "text_codepoint_ref_len"), | |
| "mean_grapheme_ref_len": _mean(bucket_rows, "text_grapheme_ref_len"), | |
| "mean_codepoint_cer": _mean(bucket_rows, "text_cer_codepoint"), | |
| "mean_grapheme_cer": _mean(bucket_rows, "text_cer_grapheme"), | |
| "mean_crop_width_px": _mean(bucket_rows, "crop_width_px"), | |
| "mean_crop_height_px": _mean(bucket_rows, "crop_height_px"), | |
| "mean_crop_aspect_ratio": _mean(bucket_rows, "crop_aspect_ratio"), | |
| "mean_reference_x_px": _mean(bucket_rows, "reference_x_px"), | |
| "mean_reference_y_px": _mean(bucket_rows, "reference_y_px"), | |
| "mean_reference_width_px": _mean(bucket_rows, "reference_width_px"), | |
| "mean_reference_height_px": _mean(bucket_rows, "reference_height_px"), | |
| "mean_reference_aspect_ratio": _mean(bucket_rows, "reference_aspect_ratio"), | |
| "mean_predicted_x_px": _mean(bucket_rows, "predicted_x_px"), | |
| "mean_predicted_y_px": _mean(bucket_rows, "predicted_y_px"), | |
| "mean_predicted_width_px": _mean(bucket_rows, "predicted_width_px"), | |
| "mean_predicted_height_px": _mean(bucket_rows, "predicted_height_px"), | |
| "mean_predicted_aspect_ratio": _mean(bucket_rows, "predicted_aspect_ratio"), | |
| "mean_crop_px_per_codepoint": _mean(bucket_rows, "crop_px_per_codepoint"), | |
| "mean_crop_px_per_grapheme": _mean(bucket_rows, "crop_px_per_grapheme"), | |
| } | |
| return stats | |
| def _mean(rows: list[dict[str, Any]], key: str) -> float | None: | |
| values = [_float(row, key) for row in rows if row.get(key) not in ("", None)] | |
| return sum(values) / len(values) if values else None | |
| def _crop_width_band(width: Any) -> str: | |
| if width in ("", None): | |
| return "missing" | |
| value = _float({"width": width}, "width", -1.0) | |
| if value < 0: | |
| return "missing" | |
| if value <= 256: | |
| return "000-256" | |
| if value <= 384: | |
| return "257-384" | |
| if value <= 512: | |
| return "385-512" | |
| if value <= 768: | |
| return "513-768" | |
| return "769-plus" | |
| def _crop_width_band_stats(rows: list[dict[str, Any]]) -> dict[str, dict[str, float | int | None]]: | |
| matched = [row for row in rows if row.get("kind") == "matched"] | |
| bands = sorted({_crop_width_band(row.get("crop_width_px")) for row in matched}) | |
| stats: dict[str, dict[str, float | int | None]] = {} | |
| for band in bands: | |
| band_rows = [row for row in matched if _crop_width_band(row.get("crop_width_px")) == band] | |
| stats[band] = { | |
| "count": len(band_rows), | |
| "mean_iou": _mean(band_rows, "iou"), | |
| "mean_confidence": _mean(band_rows, "confidence"), | |
| "mean_codepoint_ref_len": _mean(band_rows, "text_codepoint_ref_len"), | |
| "mean_codepoint_cer": _mean(band_rows, "text_cer_codepoint"), | |
| "mean_grapheme_cer": _mean(band_rows, "text_cer_grapheme"), | |
| "mean_crop_aspect_ratio": _mean(band_rows, "crop_aspect_ratio"), | |
| } | |
| return stats | |
| def _text_codepoint_len(value: Any) -> int: | |
| if value in ("", None): | |
| return 0 | |
| return len(str(value)) | |
| def _codepoint_error_mass(rows: list[dict[str, Any]]) -> dict[str, Any]: | |
| sources = { | |
| "matched_good_geometry_bad_text": 0, | |
| "matched_good_geometry_acceptable_text": 0, | |
| "matched_low_geometry_bad_text": 0, | |
| "matched_low_geometry_acceptable_text": 0, | |
| "missed_reference_text": 0, | |
| "extra_prediction_text": 0, | |
| "other": 0, | |
| } | |
| for row in rows: | |
| kind = row.get("kind") | |
| if kind == "matched": | |
| bucket = str(row.get("bucket") or "other") | |
| sources[bucket if bucket in sources else "other"] += _int(row, "text_codepoint_edits") | |
| elif kind == "reference_unmatched": | |
| ref_len = _int(row, "text_codepoint_ref_len") | |
| sources["missed_reference_text"] += ref_len if ref_len > 0 else _text_codepoint_len(row.get("reference_text")) | |
| elif kind == "prediction_unmatched": | |
| sources["extra_prediction_text"] += _text_codepoint_len(row.get("predicted_text")) | |
| else: | |
| sources["other"] += _int(row, "text_codepoint_edits") | |
| total = sum(sources.values()) | |
| fractions = {key: value / total if total else 0.0 for key, value in sources.items()} | |
| dominant_source = max(sources, key=lambda key: (sources[key], key)) if total else None | |
| return { | |
| "total": total, | |
| "by_source": sources, | |
| "fractions": fractions, | |
| "dominant_source": dominant_source, | |
| } | |
| def _spatial_group_key(row: dict[str, Any]) -> tuple[str, str]: | |
| return str(row.get("sample_id") or ""), str(row.get("page_index") or "") | |
| def _reference_spatial_context(rows: list[dict[str, Any]]) -> dict[tuple[str, str], dict[str, float]]: | |
| context: dict[tuple[str, str], dict[str, float]] = {} | |
| for row in rows: | |
| if row.get("kind") not in {"matched", "reference_unmatched"}: | |
| continue | |
| center_x = row.get("reference_center_x_px") | |
| center_y = row.get("reference_center_y_px") | |
| if center_x in ("", None) or center_y in ("", None): | |
| continue | |
| x = float(center_x) | |
| y = float(center_y) | |
| key = _spatial_group_key(row) | |
| if key not in context: | |
| context[key] = {"min_x": x, "max_x": x, "min_y": y, "max_y": y} | |
| continue | |
| item = context[key] | |
| item["min_x"] = min(item["min_x"], x) | |
| item["max_x"] = max(item["max_x"], x) | |
| item["min_y"] = min(item["min_y"], y) | |
| item["max_y"] = max(item["max_y"], y) | |
| return context | |
| def _axis_band(value: Any, *, minimum: float, maximum: float, labels: tuple[str, str, str]) -> str: | |
| if value in ("", None): | |
| return "unknown" | |
| span = maximum - minimum | |
| if span <= 0: | |
| return labels[1] | |
| ratio = (float(value) - minimum) / span | |
| if ratio < 1 / 3: | |
| return labels[0] | |
| if ratio < 2 / 3: | |
| return labels[1] | |
| return labels[2] | |
| def _empty_spatial_bucket() -> dict[str, float | int | None]: | |
| return { | |
| "reference_count": 0, | |
| "unmatched_reference_count": 0, | |
| "unmatched_reference_fraction": 0.0, | |
| "codepoint_error_mass": 0, | |
| "mean_reference_x_px": None, | |
| "mean_reference_y_px": None, | |
| "mean_reference_width_px": None, | |
| "mean_reference_height_px": None, | |
| } | |
| def _finalize_spatial_bucket(rows: list[dict[str, Any]], unmatched_rows: list[dict[str, Any]]) -> dict[str, float | int | None]: | |
| reference_count = len(rows) | |
| unmatched_count = len(unmatched_rows) | |
| return { | |
| "reference_count": reference_count, | |
| "unmatched_reference_count": unmatched_count, | |
| "unmatched_reference_fraction": unmatched_count / reference_count if reference_count else 0.0, | |
| "codepoint_error_mass": sum( | |
| _int(row, "text_codepoint_ref_len") | |
| if _int(row, "text_codepoint_ref_len") > 0 | |
| else _text_codepoint_len(row.get("reference_text")) | |
| for row in unmatched_rows | |
| ), | |
| "mean_reference_x_px": _mean(unmatched_rows, "reference_x_px"), | |
| "mean_reference_y_px": _mean(unmatched_rows, "reference_y_px"), | |
| "mean_reference_width_px": _mean(unmatched_rows, "reference_width_px"), | |
| "mean_reference_height_px": _mean(unmatched_rows, "reference_height_px"), | |
| } | |
| def _reference_spatial_stats(rows: list[dict[str, Any]]) -> dict[str, Any]: | |
| context = _reference_spatial_context(rows) | |
| reference_rows: list[dict[str, Any]] = [] | |
| for row in rows: | |
| if row.get("kind") not in {"matched", "reference_unmatched"}: | |
| continue | |
| group = context.get(_spatial_group_key(row)) | |
| if group is None: | |
| continue | |
| y_band = _axis_band( | |
| row.get("reference_center_y_px"), | |
| minimum=group["min_y"], | |
| maximum=group["max_y"], | |
| labels=("top", "middle", "bottom"), | |
| ) | |
| x_band = _axis_band( | |
| row.get("reference_center_x_px"), | |
| minimum=group["min_x"], | |
| maximum=group["max_x"], | |
| labels=("left", "center", "right"), | |
| ) | |
| enriched = dict(row) | |
| enriched["reference_y_band"] = y_band | |
| enriched["reference_x_band"] = x_band | |
| enriched["reference_xy_band"] = f"{y_band}/{x_band}" | |
| reference_rows.append(enriched) | |
| def grouped(field: str, ordered_keys: tuple[str, ...] | None = None) -> dict[str, dict[str, float | int | None]]: | |
| keys = list(ordered_keys or tuple(sorted({str(row[field]) for row in reference_rows}))) | |
| stats: dict[str, dict[str, float | int | None]] = {} | |
| for key in keys: | |
| bucket_rows = [row for row in reference_rows if row[field] == key] | |
| unmatched_rows = [row for row in bucket_rows if row.get("kind") == "reference_unmatched"] | |
| stats[key] = _finalize_spatial_bucket(bucket_rows, unmatched_rows) if bucket_rows else _empty_spatial_bucket() | |
| return stats | |
| xy_keys = tuple( | |
| f"{y}/{x}" | |
| for y in ("top", "middle", "bottom") | |
| for x in ("left", "center", "right") | |
| ) | |
| by_y_band = grouped("reference_y_band", ("top", "middle", "bottom", "unknown")) | |
| by_x_band = grouped("reference_x_band", ("left", "center", "right", "unknown")) | |
| by_xy_band = grouped("reference_xy_band", (*xy_keys, "unknown/unknown")) | |
| top_unmatched_bands = sorted( | |
| ( | |
| {"band": band, **stats} | |
| for band, stats in by_xy_band.items() | |
| if int(stats["unmatched_reference_count"] or 0) > 0 | |
| ), | |
| key=lambda row: ( | |
| int(row["codepoint_error_mass"] or 0), | |
| int(row["unmatched_reference_count"] or 0), | |
| float(row["unmatched_reference_fraction"] or 0.0), | |
| str(row["band"]), | |
| ), | |
| reverse=True, | |
| ) | |
| return { | |
| "reference_count": len(reference_rows), | |
| "unmatched_reference_count": sum(row.get("kind") == "reference_unmatched" for row in reference_rows), | |
| "by_y_band": by_y_band, | |
| "by_x_band": by_x_band, | |
| "by_xy_band": by_xy_band, | |
| "top_unmatched_bands": top_unmatched_bands, | |
| } | |
| def _unmatched_reference_overlap_stats(rows: list[dict[str, Any]]) -> dict[str, Any]: | |
| unmatched = [row for row in rows if row.get("kind") == "reference_unmatched"] | |
| source_counts: dict[str, int] = {} | |
| for row in unmatched: | |
| source = str(row.get("best_any_prediction_kind") or "none") | |
| source_counts[source] = source_counts.get(source, 0) + 1 | |
| return { | |
| "reference_unmatched_count": len(unmatched), | |
| "best_any_prediction_iou_ge_010": sum(_float(row, "best_any_prediction_iou") >= 0.10 for row in unmatched), | |
| "best_any_prediction_iou_ge_030": sum(_float(row, "best_any_prediction_iou") >= 0.30 for row in unmatched), | |
| "best_any_prediction_iou_ge_050": sum(_float(row, "best_any_prediction_iou") >= 0.50 for row in unmatched), | |
| "best_any_prediction_iou_ge_085": sum(_float(row, "best_any_prediction_iou") >= 0.85 for row in unmatched), | |
| "best_any_prediction_source_counts": dict(sorted(source_counts.items())), | |
| "mean_best_any_prediction_iou": _mean(unmatched, "best_any_prediction_iou"), | |
| } | |
| def _top_failures(rows: list[dict[str, Any]], limit: int) -> list[dict[str, Any]]: | |
| candidates = [row for row in rows if row["kind"] == "matched"] | |
| candidates.sort( | |
| key=lambda row: ( | |
| _int(row, "text_codepoint_edits"), | |
| _float(row, "text_cer_grapheme"), | |
| _float(row, "text_cer_codepoint"), | |
| ), | |
| reverse=True, | |
| ) | |
| return candidates[:limit] | |
| def _top_unmatched_references(rows: list[dict[str, Any]], limit: int) -> list[dict[str, Any]]: | |
| candidates = [row for row in rows if row["kind"] == "reference_unmatched"] | |
| candidates.sort( | |
| key=lambda row: ( | |
| _int(row, "text_codepoint_ref_len"), | |
| _float(row, "best_any_prediction_iou"), | |
| _float(row, "reference_width_px") * _float(row, "reference_height_px"), | |
| _float(row, "reference_y_px"), | |
| _float(row, "reference_x_px"), | |
| ), | |
| reverse=True, | |
| ) | |
| return candidates[:limit] | |
| def _top_unmatched_predictions(rows: list[dict[str, Any]], limit: int) -> list[dict[str, Any]]: | |
| candidates = [row for row in rows if row["kind"] == "prediction_unmatched"] | |
| candidates.sort( | |
| key=lambda row: ( | |
| _float(row, "confidence", -1.0), | |
| len(str(row.get("predicted_text") or "")), | |
| _float(row, "predicted_width_px") * _float(row, "predicted_height_px"), | |
| ), | |
| reverse=True, | |
| ) | |
| return candidates[:limit] | |
| def _near_miss_unmatched_pairs(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: | |
| references = [row for row in rows if row["kind"] == "reference_unmatched"] | |
| predictions = [row for row in rows if row["kind"] == "prediction_unmatched"] | |
| pairs: list[dict[str, Any]] = [] | |
| for reference in references: | |
| sample_id = reference.get("sample_id") | |
| page_index = reference.get("page_index") | |
| candidates = [ | |
| prediction | |
| for prediction in predictions | |
| if prediction.get("sample_id") == sample_id and prediction.get("page_index") == page_index | |
| ] | |
| best_prediction: dict[str, Any] | None = None | |
| best_iou = 0.0 | |
| for prediction in candidates: | |
| iou = _bbox_iou(reference.get("reference_bbox"), prediction.get("predicted_bbox")) | |
| if iou > best_iou: | |
| best_iou = iou | |
| best_prediction = prediction | |
| if best_prediction is None: | |
| continue | |
| pairs.append( | |
| { | |
| "sample_id": sample_id, | |
| "page_index": page_index, | |
| "reference_line_id": reference.get("reference_line_id") or "", | |
| "predicted_line_id": best_prediction.get("predicted_line_id") or "", | |
| "iou": best_iou, | |
| "reference_bbox": reference.get("reference_bbox") or "", | |
| "predicted_bbox": best_prediction.get("predicted_bbox") or "", | |
| "reference_text": reference.get("reference_text") or "", | |
| "predicted_text": best_prediction.get("predicted_text") or "", | |
| "confidence": best_prediction.get("confidence"), | |
| "reference_width_px": reference.get("reference_width_px"), | |
| "reference_height_px": reference.get("reference_height_px"), | |
| "predicted_width_px": best_prediction.get("predicted_width_px"), | |
| "predicted_height_px": best_prediction.get("predicted_height_px"), | |
| "crop_path": best_prediction.get("crop_path") or "", | |
| "crop_sha256": best_prediction.get("crop_sha256") or "", | |
| } | |
| ) | |
| pairs.sort(key=lambda row: _float(row, "iou"), reverse=True) | |
| return pairs | |
| def _write_rows(path: Path, rows: list[dict[str, Any]]) -> None: | |
| fields = [ | |
| "sample_id", | |
| "kind", | |
| "bucket", | |
| "page_index", | |
| "reference_line_id", | |
| "predicted_line_id", | |
| "iou", | |
| "text_codepoint_edits", | |
| "text_codepoint_ref_len", | |
| "text_cer_codepoint", | |
| "text_grapheme_edits", | |
| "text_grapheme_ref_len", | |
| "text_cer_grapheme", | |
| "confidence", | |
| "crop_path", | |
| "crop_sha256", | |
| "reference_bbox", | |
| "predicted_bbox", | |
| "crop_xyxy", | |
| "detected_bbox", | |
| "detected_area", | |
| "crop_width_px", | |
| "crop_height_px", | |
| "crop_aspect_ratio", | |
| "detected_width_px", | |
| "detected_height_px", | |
| "detected_aspect_ratio", | |
| "reference_width_px", | |
| "reference_height_px", | |
| "reference_x_px", | |
| "reference_y_px", | |
| "reference_center_x_px", | |
| "reference_center_y_px", | |
| "reference_aspect_ratio", | |
| "predicted_width_px", | |
| "predicted_height_px", | |
| "predicted_x_px", | |
| "predicted_y_px", | |
| "predicted_center_x_px", | |
| "predicted_center_y_px", | |
| "predicted_aspect_ratio", | |
| "best_any_prediction_iou", | |
| "best_any_prediction_line_id", | |
| "best_any_prediction_kind", | |
| "best_any_prediction_bbox", | |
| "best_any_prediction_confidence", | |
| "crop_px_per_codepoint", | |
| "crop_px_per_grapheme", | |
| "reference_text", | |
| "predicted_text", | |
| "best_any_prediction_text", | |
| ] | |
| with path.open("w", encoding="utf-8", newline="") as handle: | |
| writer = csv.DictWriter(handle, fieldnames=fields) | |
| writer.writeheader() | |
| for row in rows: | |
| payload = dict(row) | |
| for key in ("crop_xyxy", "detected_bbox"): | |
| if isinstance(payload.get(key), list): | |
| payload[key] = json.dumps(payload[key], ensure_ascii=False) | |
| writer.writerow({field: payload.get(field, "") for field in fields}) | |
| def _write_near_miss_pairs(path: Path, rows: list[dict[str, Any]]) -> None: | |
| fields = [ | |
| "sample_id", | |
| "page_index", | |
| "reference_line_id", | |
| "predicted_line_id", | |
| "iou", | |
| "confidence", | |
| "reference_bbox", | |
| "predicted_bbox", | |
| "reference_width_px", | |
| "reference_height_px", | |
| "best_any_prediction_iou", | |
| "best_any_prediction_line_id", | |
| "best_any_prediction_kind", | |
| "best_any_prediction_bbox", | |
| "best_any_prediction_confidence", | |
| "predicted_width_px", | |
| "predicted_height_px", | |
| "crop_path", | |
| "crop_sha256", | |
| "reference_text", | |
| "predicted_text", | |
| ] | |
| with path.open("w", encoding="utf-8", newline="") as handle: | |
| writer = csv.DictWriter(handle, fieldnames=fields) | |
| writer.writeheader() | |
| for row in rows: | |
| writer.writerow({field: row.get(field, "") for field in fields}) | |
| def _write_report(path: Path, result: dict[str, Any]) -> None: | |
| summary = result["summary"] | |
| buckets = summary["bucket_counts"] | |
| lines = [ | |
| "# Page Line Failure Analysis", | |
| "", | |
| "This report joins page line-alignment rows with crop provenance. It is diagnostic evidence, not a claim-safe score.", | |
| "", | |
| f"- line alignment: `{result['line_alignment_csv']}`", | |
| f"- crop manifest: `{result.get('crop_manifest') or ''}`", | |
| f"- rows: {summary['row_count']}", | |
| f"- matched rows: {summary['matched_count']}", | |
| f"- unmatched references: {summary['reference_unmatched_count']}", | |
| f"- unmatched predictions: {summary['prediction_unmatched_count']}", | |
| f"- high-IoU threshold: {result['thresholds']['high_iou']}", | |
| f"- bad codepoint CER threshold: {result['thresholds']['bad_codepoint_cer']}", | |
| f"- bad grapheme CER threshold: {result['thresholds']['bad_grapheme_cer']}", | |
| f"- high-IoU bad-text matched rows: {buckets.get('matched_good_geometry_bad_text', 0)}", | |
| f"- high-IoU acceptable-text matched rows: {buckets.get('matched_good_geometry_acceptable_text', 0)}", | |
| f"- mean matched IoU: {summary['matched_mean_iou']:.6f}", | |
| f"- mean matched codepoint CER: {summary['matched_mean_codepoint_cer']:.6f}", | |
| f"- mean matched grapheme CER: {summary['matched_mean_grapheme_cer']:.6f}", | |
| f"- mean unmatched-reference y: {_format_optional_float(summary['bucket_stats'].get('reference_unmatched', {}).get('mean_reference_y_px'))}", | |
| f"- mean unmatched-prediction confidence: {_format_optional_float(summary['bucket_stats'].get('prediction_unmatched', {}).get('mean_confidence'))}", | |
| f"- unmatched reference/prediction near-miss pairs at IoU >= 0.30: {summary['near_miss_unmatched_pair_count_iou_ge_030']}", | |
| f"- unmatched reference/prediction near-miss pairs at IoU >= 0.50: {summary['near_miss_unmatched_pair_count_iou_ge_050']}", | |
| f"- unmatched references with any prediction IoU >= 0.30: {summary['unmatched_reference_overlap_stats']['best_any_prediction_iou_ge_030']}", | |
| f"- unmatched references with any prediction IoU >= 0.50: {summary['unmatched_reference_overlap_stats']['best_any_prediction_iou_ge_050']}", | |
| f"- unmatched references with no overlapping prediction: {summary['unmatched_reference_overlap_stats']['best_any_prediction_source_counts'].get('none', 0)}", | |
| f"- dominant codepoint error source: {summary['codepoint_error_mass']['dominant_source'] or 'n/a'}", | |
| f"- total attributed codepoint error mass: {summary['codepoint_error_mass']['total']}", | |
| "", | |
| "## Geometry Signal", | |
| "", | |
| f"- mean high-IoU bad-text crop width: {_format_optional_float(summary['bucket_stats'].get('matched_good_geometry_bad_text', {}).get('mean_crop_width_px'))}", | |
| f"- mean high-IoU acceptable-text crop width: {_format_optional_float(summary['bucket_stats'].get('matched_good_geometry_acceptable_text', {}).get('mean_crop_width_px'))}", | |
| f"- mean high-IoU bad-text crop aspect ratio: {_format_optional_float(summary['bucket_stats'].get('matched_good_geometry_bad_text', {}).get('mean_crop_aspect_ratio'))}", | |
| f"- mean high-IoU acceptable-text crop aspect ratio: {_format_optional_float(summary['bucket_stats'].get('matched_good_geometry_acceptable_text', {}).get('mean_crop_aspect_ratio'))}", | |
| f"- mean unmatched-reference width: {_format_optional_float(summary['bucket_stats'].get('reference_unmatched', {}).get('mean_reference_width_px'))}", | |
| f"- mean unmatched-reference height: {_format_optional_float(summary['bucket_stats'].get('reference_unmatched', {}).get('mean_reference_height_px'))}", | |
| f"- mean unmatched-prediction width: {_format_optional_float(summary['bucket_stats'].get('prediction_unmatched', {}).get('mean_predicted_width_px'))}", | |
| f"- mean unmatched-prediction height: {_format_optional_float(summary['bucket_stats'].get('prediction_unmatched', {}).get('mean_predicted_height_px'))}", | |
| "", | |
| "## Missed Reference Spatial Bands", | |
| "", | |
| ] | |
| spatial = summary["unmatched_reference_spatial_stats"] | |
| for band, stats in spatial["by_y_band"].items(): | |
| lines.append( | |
| "- " | |
| f"{band}: {stats['unmatched_reference_count']}/{stats['reference_count']} unmatched " | |
| f"({float(stats['unmatched_reference_fraction']):.3f}), " | |
| f"error mass {stats['codepoint_error_mass']}" | |
| ) | |
| lines.extend(["", "By horizontal band:", ""]) | |
| for band, stats in spatial["by_x_band"].items(): | |
| lines.append( | |
| "- " | |
| f"{band}: {stats['unmatched_reference_count']}/{stats['reference_count']} unmatched " | |
| f"({float(stats['unmatched_reference_fraction']):.3f}), " | |
| f"error mass {stats['codepoint_error_mass']}" | |
| ) | |
| lines.extend(["", "Top combined bands:", ""]) | |
| for row in spatial["top_unmatched_bands"][:5]: | |
| lines.append( | |
| "- " | |
| f"{row['band']}: {row['unmatched_reference_count']}/{row['reference_count']} unmatched " | |
| f"({float(row['unmatched_reference_fraction']):.3f}), " | |
| f"error mass {row['codepoint_error_mass']}" | |
| ) | |
| lines.extend([ | |
| "", | |
| "## Error Mass", | |
| "", | |
| ]) | |
| for source, count in summary["codepoint_error_mass"]["by_source"].items(): | |
| fraction = summary["codepoint_error_mass"]["fractions"].get(source, 0.0) | |
| lines.append(f"- {source}: {count} ({fraction:.3f})") | |
| lines.extend([ | |
| "", | |
| "## Top Failures", | |
| "", | |
| ]) | |
| for row in result["top_failures"][:10]: | |
| lines.append( | |
| "- " | |
| f"{row['reference_line_id']} -> {row['predicted_line_id']}: " | |
| f"IoU {row['iou']:.3f}, cp CER {row['text_cer_codepoint']:.3f}, " | |
| f"gr CER {row['text_cer_grapheme']:.3f}, crop `{row['crop_path']}`" | |
| ) | |
| lines.extend(["", "## Top Unmatched References", ""]) | |
| for row in result.get("top_unmatched_references", [])[:10]: | |
| lines.append( | |
| "- " | |
| f"{row['reference_line_id']}: ref len {row['text_codepoint_ref_len']}, " | |
| f"bbox `{row['reference_bbox']}`, " | |
| f"best any prediction IoU {_format_optional_float(row.get('best_any_prediction_iou'))} " | |
| f"({row.get('best_any_prediction_kind') or 'none'} {row.get('best_any_prediction_line_id') or ''})" | |
| ) | |
| lines.extend(["", "## Top Unmatched Predictions", ""]) | |
| for row in result.get("top_unmatched_predictions", [])[:10]: | |
| lines.append( | |
| "- " | |
| f"{row['predicted_line_id']}: confidence {_format_optional_float(row.get('confidence'))}, " | |
| f"bbox `{row['predicted_bbox']}`, text `{row['predicted_text']}`" | |
| ) | |
| lines.extend(["", "## Top Unmatched Near Misses", ""]) | |
| for row in result.get("near_miss_unmatched_pairs", [])[:10]: | |
| lines.append( | |
| "- " | |
| f"{row['reference_line_id']} -> {row['predicted_line_id']}: " | |
| f"IoU {row['iou']:.3f}, confidence {_format_optional_float(row.get('confidence'))}, " | |
| f"crop `{row['crop_path']}`" | |
| ) | |
| path.write_text("\n".join(lines) + "\n", encoding="utf-8") | |
| def _format_optional_float(value: float | int | None) -> str: | |
| if value is None: | |
| return "n/a" | |
| return f"{float(value):.6f}" | |
| def analyze(args: argparse.Namespace) -> dict[str, Any]: | |
| if args.high_iou_threshold <= 0 or args.high_iou_threshold > 1: | |
| raise ValueError("--high-iou-threshold must be in (0, 1]") | |
| if args.bad_codepoint_cer < 0 or args.bad_grapheme_cer < 0: | |
| raise ValueError("bad CER thresholds must be non-negative") | |
| args.out.mkdir(parents=True, exist_ok=True) | |
| alignment_rows = _read_csv(args.line_alignment_csv) | |
| crop_rows = _read_jsonl(args.crop_manifest) | |
| joined = _joined_rows( | |
| alignment_rows, | |
| crop_rows, | |
| high_iou_threshold=args.high_iou_threshold, | |
| bad_codepoint_cer=args.bad_codepoint_cer, | |
| bad_grapheme_cer=args.bad_grapheme_cer, | |
| ) | |
| _annotate_unmatched_reference_prediction_overlaps(joined) | |
| matched = [row for row in joined if row["kind"] == "matched"] | |
| top = _top_failures(joined, args.top_k) | |
| top_unmatched_references = _top_unmatched_references(joined, args.top_k) | |
| top_unmatched_predictions = _top_unmatched_predictions(joined, args.top_k) | |
| near_miss_unmatched_pairs = _near_miss_unmatched_pairs(joined) | |
| bucket_counts = _bucket_counts(joined) | |
| bucket_stats = _bucket_stats(joined) | |
| crop_width_band_stats = _crop_width_band_stats(joined) | |
| codepoint_error_mass = _codepoint_error_mass(joined) | |
| unmatched_reference_spatial_stats = _reference_spatial_stats(joined) | |
| unmatched_reference_overlap_stats = _unmatched_reference_overlap_stats(joined) | |
| result = { | |
| "created_at": datetime.now(timezone.utc).isoformat(), | |
| "machine": platform.node(), | |
| "tool": "tools/analyze_page_line_failures.py", | |
| "line_alignment_csv": str(args.line_alignment_csv), | |
| "line_alignment_csv_sha256": sha256_file(args.line_alignment_csv), | |
| "crop_manifest": str(args.crop_manifest) if args.crop_manifest else None, | |
| "crop_manifest_sha256": sha256_file(args.crop_manifest) if args.crop_manifest else None, | |
| "thresholds": { | |
| "high_iou": args.high_iou_threshold, | |
| "bad_codepoint_cer": args.bad_codepoint_cer, | |
| "bad_grapheme_cer": args.bad_grapheme_cer, | |
| }, | |
| "summary": { | |
| "row_count": len(joined), | |
| "matched_count": len(matched), | |
| "reference_unmatched_count": sum(row["kind"] == "reference_unmatched" for row in joined), | |
| "prediction_unmatched_count": sum(row["kind"] == "prediction_unmatched" for row in joined), | |
| "bucket_counts": bucket_counts, | |
| "bucket_stats": bucket_stats, | |
| "crop_width_band_stats": crop_width_band_stats, | |
| "codepoint_error_mass": codepoint_error_mass, | |
| "unmatched_reference_spatial_stats": unmatched_reference_spatial_stats, | |
| "unmatched_reference_overlap_stats": unmatched_reference_overlap_stats, | |
| "matched_mean_iou": _mean(matched, "iou") or 0.0, | |
| "matched_mean_codepoint_cer": _mean(matched, "text_cer_codepoint") or 0.0, | |
| "matched_mean_grapheme_cer": _mean(matched, "text_cer_grapheme") or 0.0, | |
| "matched_good_geometry_bad_text_fraction": bucket_counts.get("matched_good_geometry_bad_text", 0) | |
| / max(len(matched), 1), | |
| "near_miss_unmatched_pair_count_iou_ge_030": sum( | |
| _float(row, "iou") >= 0.30 for row in near_miss_unmatched_pairs | |
| ), | |
| "near_miss_unmatched_pair_count_iou_ge_050": sum( | |
| _float(row, "iou") >= 0.50 for row in near_miss_unmatched_pairs | |
| ), | |
| }, | |
| "top_failures": top, | |
| "top_unmatched_references": top_unmatched_references, | |
| "top_unmatched_predictions": top_unmatched_predictions, | |
| "near_miss_unmatched_pairs": near_miss_unmatched_pairs[: args.top_k], | |
| } | |
| rows_path = args.out / "line-failures.csv" | |
| top_path = args.out / "top-line-failures.csv" | |
| top_unmatched_references_path = args.out / "top-unmatched-references.csv" | |
| top_unmatched_predictions_path = args.out / "top-unmatched-predictions.csv" | |
| near_miss_unmatched_pairs_path = args.out / "near-miss-unmatched-pairs.csv" | |
| result_path = args.out / "line-failure-analysis.json" | |
| report_path = args.out / "line-failure-analysis.md" | |
| _write_rows(rows_path, joined) | |
| _write_rows(top_path, top) | |
| _write_rows(top_unmatched_references_path, top_unmatched_references) | |
| _write_rows(top_unmatched_predictions_path, top_unmatched_predictions) | |
| _write_near_miss_pairs(near_miss_unmatched_pairs_path, near_miss_unmatched_pairs) | |
| result["rows_csv"] = str(rows_path) | |
| result["top_failures_csv"] = str(top_path) | |
| result["top_unmatched_references_csv"] = str(top_unmatched_references_path) | |
| result["top_unmatched_predictions_csv"] = str(top_unmatched_predictions_path) | |
| result["near_miss_unmatched_pairs_csv"] = str(near_miss_unmatched_pairs_path) | |
| result["result_path"] = str(result_path) | |
| result["report_path"] = str(report_path) | |
| result_path.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") | |
| _write_report(report_path, result) | |
| return result | |
| def main(argv: list[str] | None = None) -> int: | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| parser.add_argument("--line-alignment-csv", type=Path, required=True) | |
| parser.add_argument("--crop-manifest", type=Path) | |
| parser.add_argument("--out", type=Path, required=True) | |
| parser.add_argument("--high-iou-threshold", type=float, default=0.85) | |
| parser.add_argument("--bad-codepoint-cer", type=float, default=0.30) | |
| parser.add_argument("--bad-grapheme-cer", type=float, default=0.50) | |
| parser.add_argument("--top-k", type=int, default=50) | |
| args = parser.parse_args(argv) | |
| result = analyze(args) | |
| summary = result["summary"] | |
| print( | |
| f"wrote {result['result_path']} matched={summary['matched_count']} " | |
| f"good_geometry_bad_text={summary['bucket_counts'].get('matched_good_geometry_bad_text', 0)} " | |
| f"dominant_error_source={summary['codepoint_error_mass']['dominant_source'] or 'n/a'}" | |
| ) | |
| return 0 | |
| if __name__ == "__main__": | |
| raise SystemExit(main()) | |