from __future__ import annotations import json from pathlib import Path import pytest from ocrtech.admission import assert_recognizer_family_not_stopped, decide_recognizer_admission, review_recognizer_failures from ocrtech.error_analysis import analyze_benchmark_errors, export_failure_manifest from ocrtech.errors import ValidationError from ocrtech.manifest import ManifestEntry, load_manifest, write_manifest def test_analyze_benchmark_errors_ranks_hard_samples_and_slice_gaps(tmp_path: Path) -> None: report = tmp_path / "report.json" report.write_text( json.dumps( [ _row("surya", "s1", ["nepali", "blur"], 0.40), _row("surya", "s2", ["english", "clean"], 0.10), _row("surya", "s3", ["nepali", "clean"], 0.25), _row("tesseract", "s1", ["nepali", "blur"], 0.60), _row("tesseract", "s2", ["english", "clean"], 0.05), _row("tesseract", "s3", ["nepali", "clean"], 0.30), ] ), encoding="utf-8", ) analysis = analyze_benchmark_errors( report, tmp_path / "analysis", system="surya", baselines=["tesseract"], metric="cer", top_n=2, ) assert analysis.sample_count == 3 payload = json.loads((tmp_path / "analysis" / "error-analysis.json").read_text(encoding="utf-8")) assert payload["baseline_summary"]["tesseract"]["pairs"] == 3 assert payload["baseline_summary"]["tesseract"]["system_worse_count"] == 1 top = json.loads((tmp_path / "analysis" / "top-failures.json").read_text(encoding="utf-8")) assert [item["sample_id"] for item in top] == ["s2", "s1"] assert top[0]["comparisons"]["tesseract"]["worse_than_baseline"] == 0.05 assert payload["top_failure_slice_counts"] == [ {"slice": "blur", "count": 1}, {"slice": "clean", "count": 1}, {"slice": "english", "count": 1}, {"slice": "nepali", "count": 1}, ] slice_summary = {item["slice"]: item for item in payload["slice_summary"]} assert slice_summary["nepali"]["sample_count"] == 2 assert slice_summary["nepali"]["system_mean"] == (0.40 + 0.25) / 2 def test_analyze_benchmark_errors_without_baseline_uses_metric_badness(tmp_path: Path) -> None: report = tmp_path / "report.json" report.write_text(json.dumps([_row("surya", "easy", ["clean"], 0.01), _row("surya", "hard", ["blur"], 0.50)]), encoding="utf-8") analyze_benchmark_errors(report, tmp_path / "analysis", system="surya", metric="cer", top_n=1) top = json.loads((tmp_path / "analysis" / "top-failures.json").read_text(encoding="utf-8")) assert [item["sample_id"] for item in top] == ["hard"] assert top[0]["worst_gap"] is None def test_export_failure_manifest_preserves_entries_and_stamps_failure_metadata(tmp_path: Path) -> None: image = tmp_path / "hard.png" image.write_bytes(b"image") source_manifest = tmp_path / "source.jsonl" write_manifest( [ ManifestEntry( sample_id="hard", dataset="fixture", split="eval", image_path=str(image), text="नेपाल total", metadata={"slices": ["table"], "reference_path": "refs/hard.json"}, ) ], source_manifest, ) failures = tmp_path / "top-failures.json" failures.write_text( json.dumps( [ { "sample_id": "hard", "sample_key": "hard", "slices": ["table"], "cer": 0.42, "badness": 0.42, "worst_gap": 0.10, "comparisons": {"surya": {"baseline_value": 0.32, "worse_than_baseline": 0.10}}, } ] ), encoding="utf-8", ) export = export_failure_manifest(source_manifest, failures, tmp_path / "hard.jsonl", split="hard_eval") entries = load_manifest(export.output_manifest) assert export.selected_count == 1 assert entries[0].sample_id == "hard" assert entries[0].text == "नेपाल total" assert entries[0].split == "hard_eval" assert entries[0].metadata["reference_path"] == "refs/hard.json" assert "hard_eval" in entries[0].metadata["slices"] assert entries[0].metadata["failure_analysis"]["rank"] == 1 assert entries[0].metadata["failure_analysis"]["metric"] == "cer" assert entries[0].metadata["failure_analysis"]["metric_value"] == 0.42 def test_decide_recognizer_admission_rejects_failed_gate_and_recommends_next_actions(tmp_path: Path) -> None: validation = tmp_path / "validation.json" validation.write_text( json.dumps( { "claim_status": "not_validated", "passed": False, "sample_count": 50, "missing_requirements": ["absolute improvement -0.1 < required 0.0"], "gates": [ { "gate": {"baseline": "tesseract", "metric": "cer", "slices": ["english", "line_crop"]}, "passed": False, "pairs": 38, "candidate_mean": 0.95, "baseline_mean": 0.03, "absolute_improvement": -0.92, "relative_improvement": -30.0, "win_count": 0, "loss_count": 38, "tie_count": 0, "reason": "absolute improvement -0.92 < required 0.0", } ], } ), encoding="utf-8", ) decision = decide_recognizer_admission(validation, tmp_path / "decision") payload = json.loads((tmp_path / "decision" / "recognizer-admission-decision.json").read_text(encoding="utf-8")) assert decision.decision == "reject" assert decision.failed_gates[0].loss_count == 38 assert payload["failed_gates"][0]["baseline"] == "tesseract" assert any("English preservation" in action for action in decision.next_actions) assert (tmp_path / "decision" / "recognizer-admission-decision.md").exists() def test_decide_recognizer_admission_admits_validated_report(tmp_path: Path) -> None: validation = tmp_path / "validation.json" validation.write_text('{"claim_status":"validated","passed":true,"sample_count":50,"gates":[]}\n', encoding="utf-8") decision = decide_recognizer_admission(validation, tmp_path / "decision") assert decision.decision == "admit" assert any("full document SOTA portfolio" in action for action in decision.next_actions) def test_review_recognizer_failures_stops_repeated_zero_win_family(tmp_path: Path) -> None: model_cards: list[Path] = [] decisions: list[Path] = [] for index in range(3): card = tmp_path / f"model-{index}.json" card.write_text( json.dumps( { "model_id": f"paddle-{index}", "backend": "paddleocr", "base_model": "PaddlePaddle/devanagari_PP-OCRv3_mobile_rec", "artifacts": [], "backend_kwargs": {}, "metrics": {}, } ), encoding="utf-8", ) model_cards.append(card) decision = tmp_path / f"decision-{index}.json" decision.write_text( json.dumps( { "validation_report": f"validation-{index}.json", "decision": "reject", "claim_status": "not_validated", "passed": False, "sample_count": 50, "failed_gates": [ { "baseline": "tesseract", "metric": "cer", "slices": ["english", "line_crop"], "pairs": 50, "win_count": 0, "loss_count": 50, "tie_count": 0, } ], } ), encoding="utf-8", ) decisions.append(decision) review = review_recognizer_failures(decisions, [], tmp_path / "review", model_cards=model_cards) payload = json.loads((tmp_path / "review" / "recognizer-failure-review.json").read_text(encoding="utf-8")) assert review.families[0].recommendation == "stop_family" assert review.families[0].total_wins == 0 assert review.families[0].rejected_count == 3 assert payload["families"][0]["failed_slices"] == ["english", "line_crop"] assert (tmp_path / "review" / "recognizer-failure-review.md").exists() def test_stopped_recognizer_family_guard_normalizes_devanagari_ppocrv3_names(tmp_path: Path) -> None: review = tmp_path / "recognizer-failure-review.json" review.write_text( json.dumps( { "families": [ { "backend": "paddleocr", "base_model": "PaddlePaddle/devanagari_PP-OCRv3_mobile_rec", "family_key": "paddleocr::PaddlePaddle/devanagari_PP-OCRv3_mobile_rec", "recommendation": "stop_family", "rejected_count": 4, "total_wins": 0, "total_losses": 545, "total_ties": 7, } ] } ), encoding="utf-8", ) with pytest.raises(ValidationError, match="recognizer family is stopped"): assert_recognizer_family_not_stopped( review, backend="paddleocr", base_model="https://paddle-model-ecology.bj.bcebos.com/paddlex/official_pretrained_model/devanagari_PP-OCRv3_mobile_rec_pretrained.pdparams", ) assert_recognizer_family_not_stopped(review, backend="hf_vision_encoder_decoder", base_model="microsoft/trocr-small-printed") def _row(baseline: str, sample_id: str, slices: list[str], cer: float) -> dict[str, object]: return { "baseline": baseline, "input_path": f"{sample_id}.png", "status": "ok", "latency_seconds": 1.0, "sample_id": sample_id, "slices": slices, "metrics": {"cer": cer}, "error": None, }