Download ocr-tech/src/ocrtech/oracle.py from prosperious/ocr-projects-archive: direct link, hf CLI and curl.
- Browser
- Download file 8.58 kB
-
https://huggingface.co/prosperious/ocr-projects-archive/resolve/main/ocr-tech/src/ocrtech/oracle.py
- Command line
-
hf download hf://prosperious/ocr-projects-archive/ocr-tech/src/ocrtech/oracle.py
-
curl -L -o oracle.py https://huggingface.co/prosperious/ocr-projects-archive/resolve/main/ocr-tech/src/ocrtech/oracle.py
8.58 kB
| """Oracle analysis over benchmark reports.""" | |
| from __future__ import annotations | |
| import json | |
| from dataclasses import dataclass | |
| from pathlib import Path | |
| from typing import Any | |
| from .benchmark import ( | |
| BenchmarkResult, | |
| BenchmarkSummary, | |
| _load_benchmark_rows, | |
| summarize_benchmark_report, | |
| ) | |
| from .errors import BenchmarkError | |
| from .validation import metric_direction, sample_key | |
| class OracleReport: | |
| benchmark_report: str | |
| oracle_name: str | |
| metric: str | |
| direction: str | |
| systems: list[str] | |
| sample_count: int | |
| chosen_counts: dict[str, int] | |
| oracle_report_path: str | |
| oracle_summary_path: str | |
| oracle_choices_path: str | |
| def to_dict(self) -> dict[str, Any]: | |
| return { | |
| "benchmark_report": self.benchmark_report, | |
| "oracle_name": self.oracle_name, | |
| "metric": self.metric, | |
| "direction": self.direction, | |
| "systems": self.systems, | |
| "sample_count": self.sample_count, | |
| "chosen_counts": self.chosen_counts, | |
| "oracle_report_path": self.oracle_report_path, | |
| "oracle_summary_path": self.oracle_summary_path, | |
| "oracle_choices_path": self.oracle_choices_path, | |
| } | |
| def build_benchmark_oracle( | |
| report_path: str | Path, | |
| output_dir: str | Path, | |
| *, | |
| metric: str = "cer", | |
| systems: list[str] | None = None, | |
| direction: str | None = None, | |
| oracle_name: str = "oracle", | |
| ) -> tuple[OracleReport, BenchmarkSummary]: | |
| rows = _load_benchmark_rows(report_path) | |
| if not rows: | |
| raise BenchmarkError("benchmark report has no rows") | |
| compare_systems = systems or sorted({str(row.get("baseline") or "") for row in rows if str(row.get("baseline") or "")}) | |
| compare_systems = [name for name in compare_systems if name] | |
| if len(compare_systems) < 2: | |
| raise BenchmarkError("benchmark oracle requires at least two systems") | |
| missing = [name for name in compare_systems if not any(row.get("baseline") == name for row in rows)] | |
| if missing: | |
| raise BenchmarkError(f"requested systems missing from benchmark report: {', '.join(missing)}") | |
| target_direction = direction or metric_direction(metric) | |
| oracle_rows, chosen_counts, choices = _select_oracle_rows(rows, compare_systems, metric, target_direction, oracle_name) | |
| if not oracle_rows: | |
| raise BenchmarkError(f"benchmark oracle found no shared comparable rows for metric {metric!r}") | |
| out = Path(output_dir) | |
| out.mkdir(parents=True, exist_ok=True) | |
| combined_rows = [dict(row) for row in rows] + [result.to_dict() for result in oracle_rows] | |
| oracle_report_path = out / "oracle-report.json" | |
| oracle_report_path.write_text(json.dumps(combined_rows, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") | |
| oracle_choices_path = out / "oracle-choices.json" | |
| oracle_choices_path.write_text(json.dumps(choices, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") | |
| summary = summarize_benchmark_report( | |
| oracle_report_path, | |
| out / "oracle-summary", | |
| candidate=oracle_name, | |
| baselines=compare_systems, | |
| metrics=[metric], | |
| ) | |
| report = OracleReport( | |
| benchmark_report=str(Path(report_path)), | |
| oracle_name=oracle_name, | |
| metric=metric, | |
| direction=target_direction, | |
| systems=compare_systems, | |
| sample_count=len(oracle_rows), | |
| chosen_counts=chosen_counts, | |
| oracle_report_path=str(oracle_report_path), | |
| oracle_summary_path=str(out / "oracle-summary" / "summary.json"), | |
| oracle_choices_path=str(oracle_choices_path), | |
| ) | |
| (out / "oracle-analysis.json").write_text(json.dumps(report.to_dict(), ensure_ascii=False, indent=2) + "\n", encoding="utf-8") | |
| lines = [ | |
| "# Benchmark Oracle", | |
| "", | |
| f"Source report: `{report_path}`", | |
| f"Oracle: `{oracle_name}`", | |
| f"Metric: `{metric}`", | |
| f"Direction: `{target_direction}`", | |
| f"Samples: `{len(oracle_rows)}`", | |
| "", | |
| "## Chosen Counts", | |
| "", | |
| "| system | chosen |", | |
| "| --- | ---: |", | |
| ] | |
| for system, count in sorted(chosen_counts.items()): | |
| lines.append(f"| {system} | {count} |") | |
| lines.extend( | |
| [ | |
| "", | |
| f"Oracle report: `{oracle_report_path}`", | |
| f"Oracle choices: `{oracle_choices_path}`", | |
| f"Oracle summary: `{out / 'oracle-summary' / 'summary.json'}`", | |
| ] | |
| ) | |
| (out / "oracle-analysis.md").write_text("\n".join(lines) + "\n", encoding="utf-8") | |
| return report, summary | |
| def _select_oracle_rows( | |
| rows: list[dict[str, Any]], | |
| systems: list[str], | |
| metric: str, | |
| direction: str, | |
| oracle_name: str, | |
| ) -> tuple[list[BenchmarkResult], dict[str, int], list[dict[str, Any]]]: | |
| rows_by_sample: dict[str, dict[str, dict[str, Any]]] = {} | |
| for row in rows: | |
| if row.get("status") != "ok": | |
| continue | |
| baseline = str(row.get("baseline") or "") | |
| if baseline not in systems: | |
| continue | |
| metrics = row.get("metrics") | |
| if not isinstance(metrics, dict) or metric not in metrics: | |
| continue | |
| sample = sample_key(row) | |
| rows_by_sample.setdefault(sample, {})[baseline] = row | |
| oracle_rows: list[BenchmarkResult] = [] | |
| chosen_counts = {system: 0 for system in systems} | |
| choices: list[dict[str, Any]] = [] | |
| for sample, sample_rows in sorted(rows_by_sample.items()): | |
| if any(system not in sample_rows for system in systems): | |
| continue | |
| chosen_system, chosen_row = _best_row(sample_rows, systems, metric, direction) | |
| chosen_counts[chosen_system] += 1 | |
| oracle_row = dict(chosen_row) | |
| oracle_row["baseline"] = oracle_name | |
| oracle_row["error"] = None | |
| metrics = dict(chosen_row.get("metrics") or {}) | |
| oracle_row["metrics"] = metrics | |
| choices.append( | |
| { | |
| "sample_key": sample, | |
| "sample_id": chosen_row.get("sample_id"), | |
| "input_path": chosen_row.get("input_path"), | |
| "chosen_system": chosen_system, | |
| "metric": metric, | |
| "metric_value": float(metrics[metric]), | |
| "direction": direction, | |
| } | |
| ) | |
| oracle_result = BenchmarkResult( | |
| baseline=str(oracle_row["baseline"]), | |
| input_path=str(oracle_row.get("input_path") or ""), | |
| status=str(oracle_row.get("status") or "ok"), | |
| latency_seconds=float(oracle_row.get("latency_seconds") or 0.0), | |
| sample_id=oracle_row.get("sample_id"), | |
| slices=list(oracle_row.get("slices") or []), | |
| metrics=metrics, | |
| error=None, | |
| ) | |
| oracle_result.slices = list(oracle_row.get("slices") or []) | |
| oracle_rows.append(oracle_result) | |
| return oracle_rows, {system: count for system, count in chosen_counts.items() if count}, choices | |
| def _best_row( | |
| sample_rows: dict[str, dict[str, Any]], | |
| systems: list[str], | |
| metric: str, | |
| direction: str, | |
| ) -> tuple[str, dict[str, Any]]: | |
| best_system = systems[0] | |
| best_row = sample_rows[best_system] | |
| best_value = float((best_row.get("metrics") or {})[metric]) | |
| for system in systems[1:]: | |
| row = sample_rows[system] | |
| value = float((row.get("metrics") or {})[metric]) | |
| if _is_better(value, best_value, direction): | |
| best_system = system | |
| best_row = row | |
| best_value = value | |
| continue | |
| if value == best_value: | |
| if _tie_break_quality(row) > _tie_break_quality(best_row): | |
| best_system = system | |
| best_row = row | |
| best_value = value | |
| elif _tie_break_quality(row) == _tie_break_quality(best_row): | |
| row_latency = float(row.get("latency_seconds") or 0.0) | |
| best_latency = float(best_row.get("latency_seconds") or 0.0) | |
| if row_latency < best_latency: | |
| best_system = system | |
| best_row = row | |
| best_value = value | |
| return best_system, best_row | |
| def _is_better(candidate: float, incumbent: float, direction: str) -> bool: | |
| if direction == "lower": | |
| return candidate < incumbent | |
| return candidate > incumbent | |
| def _tie_break_quality(row: dict[str, Any]) -> float: | |
| metrics = row.get("metrics") or {} | |
| value = metrics.get("quality_score") | |
| if isinstance(value, int | float): | |
| return float(value) | |
| return float("-inf") | |