Add 195 files (runs/hf-recognizer-mixed-v2-full/finalized/package/artifacts/model/model.safetensors ...)
1ef1de7 verified Download tests/test_error_analysis.py from prosperious/ocr-tech-k2: direct link, hf CLI and curl.
- Browser
- Download file 10.5 kB
-
https://huggingface.co/prosperious/ocr-tech-k2/resolve/main/tests/test_error_analysis.py
- Command line
-
hf download hf://prosperious/ocr-tech-k2/tests/test_error_analysis.py
-
curl -L -o test_error_analysis.py https://huggingface.co/prosperious/ocr-tech-k2/resolve/main/tests/test_error_analysis.py
10.5 kB
| from __future__ import annotations | |
| import json | |
| from pathlib import Path | |
| import pytest | |
| from ocrtech.admission import assert_recognizer_family_not_stopped, decide_recognizer_admission, review_recognizer_failures | |
| from ocrtech.error_analysis import analyze_benchmark_errors, export_failure_manifest | |
| from ocrtech.errors import ValidationError | |
| from ocrtech.manifest import ManifestEntry, load_manifest, write_manifest | |
| def test_analyze_benchmark_errors_ranks_hard_samples_and_slice_gaps(tmp_path: Path) -> None: | |
| report = tmp_path / "report.json" | |
| report.write_text( | |
| json.dumps( | |
| [ | |
| _row("surya", "s1", ["nepali", "blur"], 0.40), | |
| _row("surya", "s2", ["english", "clean"], 0.10), | |
| _row("surya", "s3", ["nepali", "clean"], 0.25), | |
| _row("tesseract", "s1", ["nepali", "blur"], 0.60), | |
| _row("tesseract", "s2", ["english", "clean"], 0.05), | |
| _row("tesseract", "s3", ["nepali", "clean"], 0.30), | |
| ] | |
| ), | |
| encoding="utf-8", | |
| ) | |
| analysis = analyze_benchmark_errors( | |
| report, | |
| tmp_path / "analysis", | |
| system="surya", | |
| baselines=["tesseract"], | |
| metric="cer", | |
| top_n=2, | |
| ) | |
| assert analysis.sample_count == 3 | |
| payload = json.loads((tmp_path / "analysis" / "error-analysis.json").read_text(encoding="utf-8")) | |
| assert payload["baseline_summary"]["tesseract"]["pairs"] == 3 | |
| assert payload["baseline_summary"]["tesseract"]["system_worse_count"] == 1 | |
| top = json.loads((tmp_path / "analysis" / "top-failures.json").read_text(encoding="utf-8")) | |
| assert [item["sample_id"] for item in top] == ["s2", "s1"] | |
| assert top[0]["comparisons"]["tesseract"]["worse_than_baseline"] == 0.05 | |
| assert payload["top_failure_slice_counts"] == [ | |
| {"slice": "blur", "count": 1}, | |
| {"slice": "clean", "count": 1}, | |
| {"slice": "english", "count": 1}, | |
| {"slice": "nepali", "count": 1}, | |
| ] | |
| slice_summary = {item["slice"]: item for item in payload["slice_summary"]} | |
| assert slice_summary["nepali"]["sample_count"] == 2 | |
| assert slice_summary["nepali"]["system_mean"] == (0.40 + 0.25) / 2 | |
| def test_analyze_benchmark_errors_without_baseline_uses_metric_badness(tmp_path: Path) -> None: | |
| report = tmp_path / "report.json" | |
| report.write_text(json.dumps([_row("surya", "easy", ["clean"], 0.01), _row("surya", "hard", ["blur"], 0.50)]), encoding="utf-8") | |
| analyze_benchmark_errors(report, tmp_path / "analysis", system="surya", metric="cer", top_n=1) | |
| top = json.loads((tmp_path / "analysis" / "top-failures.json").read_text(encoding="utf-8")) | |
| assert [item["sample_id"] for item in top] == ["hard"] | |
| assert top[0]["worst_gap"] is None | |
| def test_export_failure_manifest_preserves_entries_and_stamps_failure_metadata(tmp_path: Path) -> None: | |
| image = tmp_path / "hard.png" | |
| image.write_bytes(b"image") | |
| source_manifest = tmp_path / "source.jsonl" | |
| write_manifest( | |
| [ | |
| ManifestEntry( | |
| sample_id="hard", | |
| dataset="fixture", | |
| split="eval", | |
| image_path=str(image), | |
| text="नेपाल total", | |
| metadata={"slices": ["table"], "reference_path": "refs/hard.json"}, | |
| ) | |
| ], | |
| source_manifest, | |
| ) | |
| failures = tmp_path / "top-failures.json" | |
| failures.write_text( | |
| json.dumps( | |
| [ | |
| { | |
| "sample_id": "hard", | |
| "sample_key": "hard", | |
| "slices": ["table"], | |
| "cer": 0.42, | |
| "badness": 0.42, | |
| "worst_gap": 0.10, | |
| "comparisons": {"surya": {"baseline_value": 0.32, "worse_than_baseline": 0.10}}, | |
| } | |
| ] | |
| ), | |
| encoding="utf-8", | |
| ) | |
| export = export_failure_manifest(source_manifest, failures, tmp_path / "hard.jsonl", split="hard_eval") | |
| entries = load_manifest(export.output_manifest) | |
| assert export.selected_count == 1 | |
| assert entries[0].sample_id == "hard" | |
| assert entries[0].text == "नेपाल total" | |
| assert entries[0].split == "hard_eval" | |
| assert entries[0].metadata["reference_path"] == "refs/hard.json" | |
| assert "hard_eval" in entries[0].metadata["slices"] | |
| assert entries[0].metadata["failure_analysis"]["rank"] == 1 | |
| assert entries[0].metadata["failure_analysis"]["metric"] == "cer" | |
| assert entries[0].metadata["failure_analysis"]["metric_value"] == 0.42 | |
| def test_decide_recognizer_admission_rejects_failed_gate_and_recommends_next_actions(tmp_path: Path) -> None: | |
| validation = tmp_path / "validation.json" | |
| validation.write_text( | |
| json.dumps( | |
| { | |
| "claim_status": "not_validated", | |
| "passed": False, | |
| "sample_count": 50, | |
| "missing_requirements": ["absolute improvement -0.1 < required 0.0"], | |
| "gates": [ | |
| { | |
| "gate": {"baseline": "tesseract", "metric": "cer", "slices": ["english", "line_crop"]}, | |
| "passed": False, | |
| "pairs": 38, | |
| "candidate_mean": 0.95, | |
| "baseline_mean": 0.03, | |
| "absolute_improvement": -0.92, | |
| "relative_improvement": -30.0, | |
| "win_count": 0, | |
| "loss_count": 38, | |
| "tie_count": 0, | |
| "reason": "absolute improvement -0.92 < required 0.0", | |
| } | |
| ], | |
| } | |
| ), | |
| encoding="utf-8", | |
| ) | |
| decision = decide_recognizer_admission(validation, tmp_path / "decision") | |
| payload = json.loads((tmp_path / "decision" / "recognizer-admission-decision.json").read_text(encoding="utf-8")) | |
| assert decision.decision == "reject" | |
| assert decision.failed_gates[0].loss_count == 38 | |
| assert payload["failed_gates"][0]["baseline"] == "tesseract" | |
| assert any("English preservation" in action for action in decision.next_actions) | |
| assert (tmp_path / "decision" / "recognizer-admission-decision.md").exists() | |
| def test_decide_recognizer_admission_admits_validated_report(tmp_path: Path) -> None: | |
| validation = tmp_path / "validation.json" | |
| validation.write_text('{"claim_status":"validated","passed":true,"sample_count":50,"gates":[]}\n', encoding="utf-8") | |
| decision = decide_recognizer_admission(validation, tmp_path / "decision") | |
| assert decision.decision == "admit" | |
| assert any("full document SOTA portfolio" in action for action in decision.next_actions) | |
| def test_review_recognizer_failures_stops_repeated_zero_win_family(tmp_path: Path) -> None: | |
| model_cards: list[Path] = [] | |
| decisions: list[Path] = [] | |
| for index in range(3): | |
| card = tmp_path / f"model-{index}.json" | |
| card.write_text( | |
| json.dumps( | |
| { | |
| "model_id": f"paddle-{index}", | |
| "backend": "paddleocr", | |
| "base_model": "PaddlePaddle/devanagari_PP-OCRv3_mobile_rec", | |
| "artifacts": [], | |
| "backend_kwargs": {}, | |
| "metrics": {}, | |
| } | |
| ), | |
| encoding="utf-8", | |
| ) | |
| model_cards.append(card) | |
| decision = tmp_path / f"decision-{index}.json" | |
| decision.write_text( | |
| json.dumps( | |
| { | |
| "validation_report": f"validation-{index}.json", | |
| "decision": "reject", | |
| "claim_status": "not_validated", | |
| "passed": False, | |
| "sample_count": 50, | |
| "failed_gates": [ | |
| { | |
| "baseline": "tesseract", | |
| "metric": "cer", | |
| "slices": ["english", "line_crop"], | |
| "pairs": 50, | |
| "win_count": 0, | |
| "loss_count": 50, | |
| "tie_count": 0, | |
| } | |
| ], | |
| } | |
| ), | |
| encoding="utf-8", | |
| ) | |
| decisions.append(decision) | |
| review = review_recognizer_failures(decisions, [], tmp_path / "review", model_cards=model_cards) | |
| payload = json.loads((tmp_path / "review" / "recognizer-failure-review.json").read_text(encoding="utf-8")) | |
| assert review.families[0].recommendation == "stop_family" | |
| assert review.families[0].total_wins == 0 | |
| assert review.families[0].rejected_count == 3 | |
| assert payload["families"][0]["failed_slices"] == ["english", "line_crop"] | |
| assert (tmp_path / "review" / "recognizer-failure-review.md").exists() | |
| def test_stopped_recognizer_family_guard_normalizes_devanagari_ppocrv3_names(tmp_path: Path) -> None: | |
| review = tmp_path / "recognizer-failure-review.json" | |
| review.write_text( | |
| json.dumps( | |
| { | |
| "families": [ | |
| { | |
| "backend": "paddleocr", | |
| "base_model": "PaddlePaddle/devanagari_PP-OCRv3_mobile_rec", | |
| "family_key": "paddleocr::PaddlePaddle/devanagari_PP-OCRv3_mobile_rec", | |
| "recommendation": "stop_family", | |
| "rejected_count": 4, | |
| "total_wins": 0, | |
| "total_losses": 545, | |
| "total_ties": 7, | |
| } | |
| ] | |
| } | |
| ), | |
| encoding="utf-8", | |
| ) | |
| with pytest.raises(ValidationError, match="recognizer family is stopped"): | |
| assert_recognizer_family_not_stopped( | |
| review, | |
| backend="paddleocr", | |
| base_model="https://paddle-model-ecology.bj.bcebos.com/paddlex/official_pretrained_model/devanagari_PP-OCRv3_mobile_rec_pretrained.pdparams", | |
| ) | |
| assert_recognizer_family_not_stopped(review, backend="hf_vision_encoder_decoder", base_model="microsoft/trocr-small-printed") | |
| def _row(baseline: str, sample_id: str, slices: list[str], cer: float) -> dict[str, object]: | |
| return { | |
| "baseline": baseline, | |
| "input_path": f"{sample_id}.png", | |
| "status": "ok", | |
| "latency_seconds": 1.0, | |
| "sample_id": sample_id, | |
| "slices": slices, | |
| "metrics": {"cer": cer}, | |
| "error": None, | |
| } | |