ocr-tech-k2 / tests /test_error_analysis.py
prosperious's picture
Add 195 files (runs/hf-recognizer-mixed-v2-full/finalized/package/artifacts/model/model.safetensors ...)
1ef1de7 verified
Raw History Blame Contribute Delete
10.5 kB
from __future__ import annotations
import json
from pathlib import Path
import pytest
from ocrtech.admission import assert_recognizer_family_not_stopped, decide_recognizer_admission, review_recognizer_failures
from ocrtech.error_analysis import analyze_benchmark_errors, export_failure_manifest
from ocrtech.errors import ValidationError
from ocrtech.manifest import ManifestEntry, load_manifest, write_manifest
def test_analyze_benchmark_errors_ranks_hard_samples_and_slice_gaps(tmp_path: Path) -> None:
report = tmp_path / "report.json"
report.write_text(
json.dumps(
[
_row("surya", "s1", ["nepali", "blur"], 0.40),
_row("surya", "s2", ["english", "clean"], 0.10),
_row("surya", "s3", ["nepali", "clean"], 0.25),
_row("tesseract", "s1", ["nepali", "blur"], 0.60),
_row("tesseract", "s2", ["english", "clean"], 0.05),
_row("tesseract", "s3", ["nepali", "clean"], 0.30),
]
),
encoding="utf-8",
)
analysis = analyze_benchmark_errors(
report,
tmp_path / "analysis",
system="surya",
baselines=["tesseract"],
metric="cer",
top_n=2,
)
assert analysis.sample_count == 3
payload = json.loads((tmp_path / "analysis" / "error-analysis.json").read_text(encoding="utf-8"))
assert payload["baseline_summary"]["tesseract"]["pairs"] == 3
assert payload["baseline_summary"]["tesseract"]["system_worse_count"] == 1
top = json.loads((tmp_path / "analysis" / "top-failures.json").read_text(encoding="utf-8"))
assert [item["sample_id"] for item in top] == ["s2", "s1"]
assert top[0]["comparisons"]["tesseract"]["worse_than_baseline"] == 0.05
assert payload["top_failure_slice_counts"] == [
{"slice": "blur", "count": 1},
{"slice": "clean", "count": 1},
{"slice": "english", "count": 1},
{"slice": "nepali", "count": 1},
]
slice_summary = {item["slice"]: item for item in payload["slice_summary"]}
assert slice_summary["nepali"]["sample_count"] == 2
assert slice_summary["nepali"]["system_mean"] == (0.40 + 0.25) / 2
def test_analyze_benchmark_errors_without_baseline_uses_metric_badness(tmp_path: Path) -> None:
report = tmp_path / "report.json"
report.write_text(json.dumps([_row("surya", "easy", ["clean"], 0.01), _row("surya", "hard", ["blur"], 0.50)]), encoding="utf-8")
analyze_benchmark_errors(report, tmp_path / "analysis", system="surya", metric="cer", top_n=1)
top = json.loads((tmp_path / "analysis" / "top-failures.json").read_text(encoding="utf-8"))
assert [item["sample_id"] for item in top] == ["hard"]
assert top[0]["worst_gap"] is None
def test_export_failure_manifest_preserves_entries_and_stamps_failure_metadata(tmp_path: Path) -> None:
image = tmp_path / "hard.png"
image.write_bytes(b"image")
source_manifest = tmp_path / "source.jsonl"
write_manifest(
[
ManifestEntry(
sample_id="hard",
dataset="fixture",
split="eval",
image_path=str(image),
text="नेपाल total",
metadata={"slices": ["table"], "reference_path": "refs/hard.json"},
)
],
source_manifest,
)
failures = tmp_path / "top-failures.json"
failures.write_text(
json.dumps(
[
{
"sample_id": "hard",
"sample_key": "hard",
"slices": ["table"],
"cer": 0.42,
"badness": 0.42,
"worst_gap": 0.10,
"comparisons": {"surya": {"baseline_value": 0.32, "worse_than_baseline": 0.10}},
}
]
),
encoding="utf-8",
)
export = export_failure_manifest(source_manifest, failures, tmp_path / "hard.jsonl", split="hard_eval")
entries = load_manifest(export.output_manifest)
assert export.selected_count == 1
assert entries[0].sample_id == "hard"
assert entries[0].text == "नेपाल total"
assert entries[0].split == "hard_eval"
assert entries[0].metadata["reference_path"] == "refs/hard.json"
assert "hard_eval" in entries[0].metadata["slices"]
assert entries[0].metadata["failure_analysis"]["rank"] == 1
assert entries[0].metadata["failure_analysis"]["metric"] == "cer"
assert entries[0].metadata["failure_analysis"]["metric_value"] == 0.42
def test_decide_recognizer_admission_rejects_failed_gate_and_recommends_next_actions(tmp_path: Path) -> None:
validation = tmp_path / "validation.json"
validation.write_text(
json.dumps(
{
"claim_status": "not_validated",
"passed": False,
"sample_count": 50,
"missing_requirements": ["absolute improvement -0.1 < required 0.0"],
"gates": [
{
"gate": {"baseline": "tesseract", "metric": "cer", "slices": ["english", "line_crop"]},
"passed": False,
"pairs": 38,
"candidate_mean": 0.95,
"baseline_mean": 0.03,
"absolute_improvement": -0.92,
"relative_improvement": -30.0,
"win_count": 0,
"loss_count": 38,
"tie_count": 0,
"reason": "absolute improvement -0.92 < required 0.0",
}
],
}
),
encoding="utf-8",
)
decision = decide_recognizer_admission(validation, tmp_path / "decision")
payload = json.loads((tmp_path / "decision" / "recognizer-admission-decision.json").read_text(encoding="utf-8"))
assert decision.decision == "reject"
assert decision.failed_gates[0].loss_count == 38
assert payload["failed_gates"][0]["baseline"] == "tesseract"
assert any("English preservation" in action for action in decision.next_actions)
assert (tmp_path / "decision" / "recognizer-admission-decision.md").exists()
def test_decide_recognizer_admission_admits_validated_report(tmp_path: Path) -> None:
validation = tmp_path / "validation.json"
validation.write_text('{"claim_status":"validated","passed":true,"sample_count":50,"gates":[]}\n', encoding="utf-8")
decision = decide_recognizer_admission(validation, tmp_path / "decision")
assert decision.decision == "admit"
assert any("full document SOTA portfolio" in action for action in decision.next_actions)
def test_review_recognizer_failures_stops_repeated_zero_win_family(tmp_path: Path) -> None:
model_cards: list[Path] = []
decisions: list[Path] = []
for index in range(3):
card = tmp_path / f"model-{index}.json"
card.write_text(
json.dumps(
{
"model_id": f"paddle-{index}",
"backend": "paddleocr",
"base_model": "PaddlePaddle/devanagari_PP-OCRv3_mobile_rec",
"artifacts": [],
"backend_kwargs": {},
"metrics": {},
}
),
encoding="utf-8",
)
model_cards.append(card)
decision = tmp_path / f"decision-{index}.json"
decision.write_text(
json.dumps(
{
"validation_report": f"validation-{index}.json",
"decision": "reject",
"claim_status": "not_validated",
"passed": False,
"sample_count": 50,
"failed_gates": [
{
"baseline": "tesseract",
"metric": "cer",
"slices": ["english", "line_crop"],
"pairs": 50,
"win_count": 0,
"loss_count": 50,
"tie_count": 0,
}
],
}
),
encoding="utf-8",
)
decisions.append(decision)
review = review_recognizer_failures(decisions, [], tmp_path / "review", model_cards=model_cards)
payload = json.loads((tmp_path / "review" / "recognizer-failure-review.json").read_text(encoding="utf-8"))
assert review.families[0].recommendation == "stop_family"
assert review.families[0].total_wins == 0
assert review.families[0].rejected_count == 3
assert payload["families"][0]["failed_slices"] == ["english", "line_crop"]
assert (tmp_path / "review" / "recognizer-failure-review.md").exists()
def test_stopped_recognizer_family_guard_normalizes_devanagari_ppocrv3_names(tmp_path: Path) -> None:
review = tmp_path / "recognizer-failure-review.json"
review.write_text(
json.dumps(
{
"families": [
{
"backend": "paddleocr",
"base_model": "PaddlePaddle/devanagari_PP-OCRv3_mobile_rec",
"family_key": "paddleocr::PaddlePaddle/devanagari_PP-OCRv3_mobile_rec",
"recommendation": "stop_family",
"rejected_count": 4,
"total_wins": 0,
"total_losses": 545,
"total_ties": 7,
}
]
}
),
encoding="utf-8",
)
with pytest.raises(ValidationError, match="recognizer family is stopped"):
assert_recognizer_family_not_stopped(
review,
backend="paddleocr",
base_model="https://paddle-model-ecology.bj.bcebos.com/paddlex/official_pretrained_model/devanagari_PP-OCRv3_mobile_rec_pretrained.pdparams",
)
assert_recognizer_family_not_stopped(review, backend="hf_vision_encoder_decoder", base_model="microsoft/trocr-small-printed")
def _row(baseline: str, sample_id: str, slices: list[str], cer: float) -> dict[str, object]:
return {
"baseline": baseline,
"input_path": f"{sample_id}.png",
"status": "ok",
"latency_seconds": 1.0,
"sample_id": sample_id,
"slices": slices,
"metrics": {"cer": cer},
"error": None,
}