File size: 11,515 Bytes
d82db76
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
#!/usr/bin/env python3
"""Audit page candidate-line insertion effects against a baseline page score."""

from __future__ import annotations

import argparse
import csv
import json
from datetime import UTC, datetime
from pathlib import Path
from typing import Any

from ocrtech.manifest import sha256_file


REPORT_JSON = "candidate-page-line-application-effect-audit.json"
INSERTED_TSV = "candidate-page-line-application-inserted-lines.tsv"

OBJECTIVE_METRICS = (
    "text_cer_codepoint_micro",
    "text_cer_grapheme_micro",
    "line_detection_f1_micro",
    "line_detection_precision_micro",
    "line_detection_recall_micro",
    "line_unmatched_reference_count",
    "line_unmatched_prediction_count",
    "line_blank_prediction_count",
    "matched_line_reading_order_pair_accuracy",
    "matched_line_text_cer_codepoint_micro",
)


def _read_json(path: Path, label: str) -> dict[str, Any]:
    try:
        payload = json.loads(path.read_text(encoding="utf-8"))
    except FileNotFoundError as exc:
        raise ValueError(f"{label} is missing: {path}") from exc
    except json.JSONDecodeError as exc:
        raise ValueError(f"{label} is invalid JSON: {path}: {exc}") from exc
    if not isinstance(payload, dict):
        raise ValueError(f"{label} must contain a JSON object: {path}")
    return payload


def _float(value: Any) -> float | None:
    if value in ("", None):
        return None
    try:
        return float(value)
    except (TypeError, ValueError):
        return None


def _metric_delta(candidate: Any, baseline: Any) -> float | None:
    candidate_float = _float(candidate)
    baseline_float = _float(baseline)
    if candidate_float is None or baseline_float is None:
        return None
    return candidate_float - baseline_float


def _read_alignment_by_prediction(path: Path) -> dict[str, dict[str, Any]]:
    try:
        handle = path.open("r", encoding="utf-8", newline="")
    except FileNotFoundError as exc:
        raise ValueError(f"line alignment CSV is missing: {path}") from exc
    rows: dict[str, dict[str, Any]] = {}
    with handle:
        reader = csv.DictReader(handle)
        if reader.fieldnames is None:
            raise ValueError(f"line alignment CSV has no header: {path}")
        for row_number, row in enumerate(reader, start=2):
            predicted_line_id = str(row.get("predicted_line_id") or "").strip()
            if not predicted_line_id:
                continue
            if predicted_line_id in rows:
                raise ValueError(f"duplicate predicted_line_id in line alignment at {path}:{row_number}: {predicted_line_id}")
            rows[predicted_line_id] = dict(row)
    return rows


def _inserted_rows(application: dict[str, Any], alignment_by_prediction: dict[str, dict[str, Any]]) -> list[dict[str, Any]]:
    applied_rows = application.get("applied_rows")
    if not isinstance(applied_rows, list):
        raise ValueError("application report missing applied_rows list")
    inserted: list[dict[str, Any]] = []
    for row in applied_rows:
        if not isinstance(row, dict):
            raise ValueError("application applied_rows entries must be objects")
        inserted_line_id = str(row.get("inserted_line_id") or "").strip()
        if not inserted_line_id:
            raise ValueError(f"applied row missing inserted_line_id: {row.get('row_id')!r}")
        alignment = alignment_by_prediction.get(inserted_line_id)
        if alignment is None:
            inserted.append(
                {
                    "row_id": row.get("row_id") or "",
                    "sample_id": row.get("sample_id") or "",
                    "candidate_line_id": row.get("candidate_line_id") or "",
                    "inserted_line_id": inserted_line_id,
                    "alignment_kind": "alignment_missing",
                    "reference_line_id": "",
                    "iou": None,
                    "text_codepoint_edits": None,
                    "text_codepoint_ref_len": None,
                    "text_cer_codepoint": None,
                    "inserted_text": row.get("inserted_text") or "",
                    "selection_policy": row.get("selection_policy") or "",
                }
            )
            continue
        inserted.append(
            {
                "row_id": row.get("row_id") or "",
                "sample_id": row.get("sample_id") or alignment.get("sample_id") or "",
                "candidate_line_id": row.get("candidate_line_id") or "",
                "inserted_line_id": inserted_line_id,
                "alignment_kind": alignment.get("kind") or "",
                "reference_line_id": alignment.get("reference_line_id") or "",
                "iou": _float(alignment.get("iou")),
                "text_codepoint_edits": _float(alignment.get("text_codepoint_edits")),
                "text_codepoint_ref_len": _float(alignment.get("text_codepoint_ref_len")),
                "text_cer_codepoint": _float(alignment.get("text_cer_codepoint")),
                "inserted_text": row.get("inserted_text") or "",
                "selection_policy": row.get("selection_policy") or "",
            }
        )
    return inserted


def _write_inserted_tsv(path: Path, rows: list[dict[str, Any]]) -> None:
    fieldnames = [
        "row_id",
        "sample_id",
        "candidate_line_id",
        "inserted_line_id",
        "alignment_kind",
        "reference_line_id",
        "iou",
        "text_codepoint_edits",
        "text_codepoint_ref_len",
        "text_cer_codepoint",
        "selection_policy",
    ]
    with path.open("w", encoding="utf-8", newline="") as handle:
        writer = csv.DictWriter(handle, delimiter="\t", fieldnames=fieldnames)
        writer.writeheader()
        for row in rows:
            writer.writerow({field: row.get(field, "") for field in fieldnames})


def _metric_summary(baseline: dict[str, Any], candidate: dict[str, Any]) -> dict[str, dict[str, Any]]:
    summary: dict[str, dict[str, Any]] = {}
    for metric in OBJECTIVE_METRICS:
        baseline_value = baseline.get(metric)
        candidate_value = candidate.get(metric)
        summary[metric] = {
            "baseline": baseline_value,
            "candidate": candidate_value,
            "delta_candidate_minus_baseline": _metric_delta(candidate_value, baseline_value),
        }
    return summary


def _book_ocr_issues(metrics: dict[str, dict[str, Any]], inserted_counts: dict[str, int]) -> list[str]:
    issues: list[str] = []
    cp_delta = metrics["text_cer_codepoint_micro"]["delta_candidate_minus_baseline"]
    if cp_delta is None or cp_delta > 0:
        issues.append(f"text_cer_codepoint_micro does not improve or tie: delta={cp_delta}")
    graph_delta = metrics["text_cer_grapheme_micro"]["delta_candidate_minus_baseline"]
    if graph_delta is None or graph_delta > 0:
        issues.append(f"text_cer_grapheme_micro does not improve or tie: delta={graph_delta}")
    line_f1_delta = metrics["line_detection_f1_micro"]["delta_candidate_minus_baseline"]
    if line_f1_delta is None or line_f1_delta < 0:
        issues.append(f"line_detection_f1_micro regresses: delta={line_f1_delta}")
    extra_delta = metrics["line_unmatched_prediction_count"]["delta_candidate_minus_baseline"]
    if extra_delta is None or extra_delta > 0:
        issues.append(f"line_unmatched_prediction_count increases: delta={extra_delta}")
    missed_delta = metrics["line_unmatched_reference_count"]["delta_candidate_minus_baseline"]
    if missed_delta is None or missed_delta > 0:
        issues.append(f"line_unmatched_reference_count increases: delta={missed_delta}")
    order_delta = metrics["matched_line_reading_order_pair_accuracy"]["delta_candidate_minus_baseline"]
    if order_delta is None or order_delta < 0:
        issues.append(f"matched_line_reading_order_pair_accuracy regresses: delta={order_delta}")
    if inserted_counts.get("prediction_unmatched", 0) > 0:
        issues.append(f"inserted candidate rows include unmatched extra predictions: {inserted_counts['prediction_unmatched']}")
    if inserted_counts.get("alignment_missing", 0) > 0:
        issues.append(f"inserted candidate rows missing from line alignment: {inserted_counts['alignment_missing']}")
    return issues


def audit_application_effect(args: argparse.Namespace) -> dict[str, Any]:
    application_path = Path(args.application_report)
    baseline_metrics_path = Path(args.baseline_score_dir) / "metrics.json"
    candidate_metrics_path = Path(args.candidate_score_dir) / "metrics.json"
    alignment_path = Path(args.candidate_score_dir) / "line-alignment.csv"
    out = Path(args.out)
    out.mkdir(parents=True, exist_ok=True)

    application = _read_json(application_path, "candidate page-line application report")
    baseline_metrics = _read_json(baseline_metrics_path, "baseline metrics")
    candidate_metrics = _read_json(candidate_metrics_path, "candidate metrics")
    alignment = _read_alignment_by_prediction(alignment_path)
    inserted = _inserted_rows(application, alignment)
    inserted_counts: dict[str, int] = {}
    for row in inserted:
        kind = str(row.get("alignment_kind") or "unknown")
        inserted_counts[kind] = inserted_counts.get(kind, 0) + 1
    metrics = _metric_summary(baseline_metrics, candidate_metrics)
    issues = _book_ocr_issues(metrics, inserted_counts)
    inserted_tsv = out / INSERTED_TSV
    _write_inserted_tsv(inserted_tsv, inserted)
    report_path = out / REPORT_JSON
    report = {
        "tool": "tools/audit_candidate_page_line_application_effect.py",
        "created_at_utc": datetime.now(UTC).isoformat(),
        "application_report": str(application_path),
        "application_report_sha256": sha256_file(application_path),
        "baseline_score_dir": str(args.baseline_score_dir),
        "baseline_metrics_sha256": sha256_file(baseline_metrics_path),
        "candidate_score_dir": str(args.candidate_score_dir),
        "candidate_metrics_sha256": sha256_file(candidate_metrics_path),
        "line_alignment_sha256": sha256_file(alignment_path),
        "inserted_line_count": len(inserted),
        "inserted_alignment_counts": inserted_counts,
        "objective_metric_comparison": metrics,
        "book_ocr_objective_safe": not issues,
        "issues": issues,
        "inserted_lines_tsv": str(inserted_tsv),
        "inserted_lines_tsv_sha256": sha256_file(inserted_tsv),
        "claim_boundary": (
            "Page-candidate application objective audit. It compares a candidate projection to a baseline "
            "on page OCR objective metrics and inserted-line alignment; it does not certify claim-grade OCR."
        ),
    }
    report["report_json"] = str(report_path)
    report_path.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
    return report


def main() -> None:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--application-report", required=True, type=Path)
    parser.add_argument("--baseline-score-dir", required=True, type=Path)
    parser.add_argument("--candidate-score-dir", required=True, type=Path)
    parser.add_argument("--out", required=True, type=Path)
    args = parser.parse_args()
    report = audit_application_effect(args)
    print(report["report_json"])
    print(
        f"book_ocr_objective_safe={report['book_ocr_objective_safe']} "
        f"inserted={report['inserted_line_count']} issues={len(report['issues'])}"
    )


if __name__ == "__main__":
    main()