from __future__ import annotations import json from pathlib import Path from ocrtech.oracle import build_benchmark_oracle def test_build_benchmark_oracle_selects_best_system_per_sample(tmp_path: Path) -> None: report = tmp_path / "report.json" report.write_text( json.dumps( [ _row("surya", "s1", 0.10, 0.90), _row("tesseract", "s1", 0.30, 0.80), _row("surya", "s2", 0.50, 0.90), _row("tesseract", "s2", 0.20, 0.70), _row("surya", "s3", 0.40, 0.80), _row("tesseract", "s3", 0.40, 0.90), ] ), encoding="utf-8", ) oracle, summary = build_benchmark_oracle( report, tmp_path / "oracle", systems=["surya", "tesseract"], metric="cer", oracle_name="oracle", ) assert oracle.sample_count == 3 assert oracle.chosen_counts == {"surya": 1, "tesseract": 2} assert (tmp_path / "oracle" / "oracle-analysis.json").exists() assert (tmp_path / "oracle" / "oracle-choices.json").exists() assert (tmp_path / "oracle" / "oracle-summary" / "summary.json").exists() oracle_report_rows = json.loads((tmp_path / "oracle" / "oracle-report.json").read_text(encoding="utf-8")) oracle_metrics = [row["metrics"] for row in oracle_report_rows if row["baseline"] == "oracle"] assert all("oracle_source_index" not in metrics for metrics in oracle_metrics) choices = json.loads((tmp_path / "oracle" / "oracle-choices.json").read_text(encoding="utf-8")) assert [choice["chosen_system"] for choice in choices] == ["surya", "tesseract", "tesseract"] aggregate = summary.aggregate_metrics assert aggregate["oracle"]["all"]["cer"] == (0.10 + 0.20 + 0.40) / 3 paired = [item for item in summary.paired_metrics if item.baseline == "surya" and item.slice_name == "all"] assert paired assert paired[0].win_count == 1 assert paired[0].loss_count == 0 assert paired[0].tie_count == 2 def _row(baseline: str, sample_id: str, cer: float, quality: float) -> dict[str, object]: return { "baseline": baseline, "input_path": f"{sample_id}.png", "status": "ok", "latency_seconds": 1.0, "sample_id": sample_id, "slices": ["all"], "metrics": {"cer": cer, "quality_score": quality}, "error": None, }