Add 1000 files (ocr-tech/tests/test_audit_limbu_book_missed_line_review_coverage.py ...)
d82db76 verified Download ocr-tech-mps-integration/src/ocrtech/calibration.py from prosperious/ocr-projects-archive: direct link, hf CLI and curl.
- Browser
- Download file 34.1 kB
-
https://huggingface.co/prosperious/ocr-projects-archive/resolve/main/ocr-tech-mps-integration/src/ocrtech/calibration.py
- Command line
-
hf download hf://prosperious/ocr-projects-archive/ocr-tech-mps-integration/src/ocrtech/calibration.py
-
curl -L -o calibration.py https://huggingface.co/prosperious/ocr-projects-archive/resolve/main/ocr-tech-mps-integration/src/ocrtech/calibration.py
34.1 kB
| """Calibration helpers for routed OCR candidates.""" | |
| from __future__ import annotations | |
| import json | |
| import math | |
| import re | |
| from dataclasses import dataclass, field | |
| from pathlib import Path | |
| from typing import Any | |
| from .errors import BenchmarkError | |
| from .experiments import ExperimentReport | |
| from .manifest import sha256_file | |
| from .models import write_model_card | |
| from .validation import metric_direction, sample_key | |
| class ThresholdTrial: | |
| threshold: float | |
| pairs: int | |
| candidate_picks: int | |
| chosen_mean: float | |
| primary_mean: float | |
| absolute_improvement: float | |
| relative_improvement: float | |
| def to_dict(self) -> dict[str, Any]: | |
| return { | |
| "threshold": self.threshold, | |
| "pairs": self.pairs, | |
| "candidate_picks": self.candidate_picks, | |
| "chosen_mean": self.chosen_mean, | |
| "primary_mean": self.primary_mean, | |
| "absolute_improvement": self.absolute_improvement, | |
| "relative_improvement": self.relative_improvement, | |
| } | |
| class CalibrationReport: | |
| experiment_report: str | |
| benchmark_report: str | |
| eval_manifest: str | |
| candidate_baseline: str | |
| primary_baseline: str | |
| metric: str | |
| direction: str | |
| selected_threshold: float | |
| selected_trial: ThresholdTrial | |
| trials: list[ThresholdTrial] = field(default_factory=list) | |
| def to_dict(self) -> dict[str, Any]: | |
| return { | |
| "experiment_report": self.experiment_report, | |
| "benchmark_report": self.benchmark_report, | |
| "eval_manifest": self.eval_manifest, | |
| "candidate_baseline": self.candidate_baseline, | |
| "primary_baseline": self.primary_baseline, | |
| "metric": self.metric, | |
| "direction": self.direction, | |
| "selected_threshold": self.selected_threshold, | |
| "selected_trial": self.selected_trial.to_dict(), | |
| "trials": [trial.to_dict() for trial in self.trials], | |
| } | |
| class ScriptThresholdTrial: | |
| threshold: float | |
| pairs: int | |
| secondary_picks: int | |
| chosen_mean: float | |
| primary_mean: float | |
| absolute_improvement: float | |
| relative_improvement: float | |
| def to_dict(self) -> dict[str, Any]: | |
| return { | |
| "threshold": self.threshold, | |
| "pairs": self.pairs, | |
| "secondary_picks": self.secondary_picks, | |
| "chosen_mean": self.chosen_mean, | |
| "primary_mean": self.primary_mean, | |
| "absolute_improvement": self.absolute_improvement, | |
| "relative_improvement": self.relative_improvement, | |
| } | |
| class ScriptCalibrationReport: | |
| experiment_report: str | |
| benchmark_report: str | |
| eval_manifest: str | |
| primary_baseline: str | |
| secondary_baseline: str | |
| metric: str | |
| direction: str | |
| script: str | |
| selected_threshold: float | |
| selected_trial: ScriptThresholdTrial | |
| trials: list[ScriptThresholdTrial] = field(default_factory=list) | |
| def to_dict(self) -> dict[str, Any]: | |
| return { | |
| "experiment_report": self.experiment_report, | |
| "benchmark_report": self.benchmark_report, | |
| "eval_manifest": self.eval_manifest, | |
| "primary_baseline": self.primary_baseline, | |
| "secondary_baseline": self.secondary_baseline, | |
| "metric": self.metric, | |
| "direction": self.direction, | |
| "script": self.script, | |
| "selected_threshold": self.selected_threshold, | |
| "selected_trial": self.selected_trial.to_dict(), | |
| "trials": [trial.to_dict() for trial in self.trials], | |
| } | |
| def calibrate_quality_router( | |
| experiment_report: str | Path, | |
| output_dir: str | Path, | |
| *, | |
| model_id: str, | |
| primary_engine: str, | |
| secondary_engine: str, | |
| primary_engine_kwargs: dict[str, Any] | None = None, | |
| secondary_engine_kwargs: dict[str, Any] | None = None, | |
| candidate_baseline: str = "candidate", | |
| primary_baseline: str = "tesseract", | |
| metric: str = "cer", | |
| direction: str | None = None, | |
| threshold_start: float = -0.20, | |
| threshold_end: float = 0.20, | |
| threshold_step: float = 0.01, | |
| base_model: str = "tesseract-ocr", | |
| notes: str | None = None, | |
| ) -> tuple[CalibrationReport, Path]: | |
| experiment = _load_experiment_report(experiment_report) | |
| report_rows = _load_benchmark_rows(experiment.benchmark_report) | |
| resolved_direction = direction or metric_direction(metric) | |
| trials = _trial_thresholds( | |
| report_rows, | |
| candidate_baseline=candidate_baseline, | |
| primary_baseline=primary_baseline, | |
| metric=metric, | |
| direction=resolved_direction, | |
| threshold_start=threshold_start, | |
| threshold_end=threshold_end, | |
| threshold_step=threshold_step, | |
| ) | |
| if not trials: | |
| raise BenchmarkError("quality router calibration produced no valid threshold trials") | |
| selected = _select_best_trial(trials, resolved_direction) | |
| calibration = CalibrationReport( | |
| experiment_report=str(Path(experiment_report)), | |
| benchmark_report=experiment.benchmark_report, | |
| eval_manifest=experiment.eval_manifest, | |
| candidate_baseline=candidate_baseline, | |
| primary_baseline=primary_baseline, | |
| metric=metric, | |
| direction=resolved_direction, | |
| selected_threshold=selected.threshold, | |
| selected_trial=selected, | |
| trials=trials, | |
| ) | |
| out = Path(output_dir) | |
| out.mkdir(parents=True, exist_ok=True) | |
| (out / "calibration.json").write_text(json.dumps(calibration.to_dict(), ensure_ascii=False, indent=2) + "\n", encoding="utf-8") | |
| (out / "calibration.md").write_text(_render_calibration_markdown(calibration), encoding="utf-8") | |
| model_path = write_model_card( | |
| out / "model-card.json", | |
| model_id=model_id, | |
| backend="quality_select_composite", | |
| base_model=base_model, | |
| backend_kwargs={ | |
| "primary_engine": primary_engine, | |
| "secondary_engine": secondary_engine, | |
| "primary_engine_kwargs": primary_engine_kwargs or {}, | |
| "secondary_engine_kwargs": secondary_engine_kwargs or {}, | |
| "secondary_quality_margin": selected.threshold, | |
| }, | |
| provenance={ | |
| "calibration": { | |
| "experiment_report": {"path": str(Path(experiment_report)), "sha256": _maybe_sha256(Path(experiment_report))}, | |
| "benchmark_report": {"path": experiment.benchmark_report, "sha256": _maybe_sha256(Path(experiment.benchmark_report))}, | |
| "eval_manifest": {"path": experiment.eval_manifest, "sha256": _maybe_sha256(Path(experiment.eval_manifest))}, | |
| "candidate_baseline": candidate_baseline, | |
| "primary_baseline": primary_baseline, | |
| "metric": metric, | |
| "direction": resolved_direction, | |
| "selected_threshold": selected.threshold, | |
| "threshold_start": threshold_start, | |
| "threshold_end": threshold_end, | |
| "threshold_step": threshold_step, | |
| } | |
| }, | |
| metrics_report=out / "calibration.json", | |
| notes=notes, | |
| ) | |
| return calibration, model_path | |
| def calibrate_script_router( | |
| experiment_report: str | Path, | |
| output_dir: str | Path, | |
| *, | |
| model_id: str, | |
| primary_engine: str, | |
| secondary_engine: str, | |
| primary_engine_kwargs: dict[str, Any] | None = None, | |
| secondary_engine_kwargs: dict[str, Any] | None = None, | |
| primary_baseline: str = "tesseract", | |
| secondary_baseline: str = "surya", | |
| metric: str = "cer", | |
| direction: str | None = None, | |
| script: str = "devanagari", | |
| threshold_start: float = 0.0, | |
| threshold_end: float = 0.6, | |
| threshold_step: float = 0.05, | |
| routing_granularity: str = "document", | |
| secondary_structure_backfill: bool = False, | |
| base_model: str = "tesseract+surya", | |
| notes: str | None = None, | |
| ) -> tuple[ScriptCalibrationReport, Path]: | |
| experiment = _load_experiment_report(experiment_report) | |
| report_rows = _load_benchmark_rows(experiment.benchmark_report) | |
| resolved_direction = direction or metric_direction(metric) | |
| trials = _trial_script_thresholds( | |
| experiment, | |
| report_rows, | |
| primary_baseline=primary_baseline, | |
| secondary_baseline=secondary_baseline, | |
| metric=metric, | |
| direction=resolved_direction, | |
| script=script, | |
| threshold_start=threshold_start, | |
| threshold_end=threshold_end, | |
| threshold_step=threshold_step, | |
| ) | |
| if not trials: | |
| raise BenchmarkError("script router calibration produced no valid threshold trials") | |
| selected = _select_best_script_trial(trials, resolved_direction) | |
| calibration = ScriptCalibrationReport( | |
| experiment_report=str(Path(experiment_report)), | |
| benchmark_report=experiment.benchmark_report, | |
| eval_manifest=experiment.eval_manifest, | |
| primary_baseline=primary_baseline, | |
| secondary_baseline=secondary_baseline, | |
| metric=metric, | |
| direction=resolved_direction, | |
| script=script, | |
| selected_threshold=selected.threshold, | |
| selected_trial=selected, | |
| trials=trials, | |
| ) | |
| out = Path(output_dir) | |
| out.mkdir(parents=True, exist_ok=True) | |
| (out / "calibration.json").write_text(json.dumps(calibration.to_dict(), ensure_ascii=False, indent=2) + "\n", encoding="utf-8") | |
| (out / "calibration.md").write_text(_render_script_calibration_markdown(calibration), encoding="utf-8") | |
| model_path = write_model_card( | |
| out / "model-card.json", | |
| model_id=model_id, | |
| backend="script_select_composite", | |
| base_model=base_model, | |
| backend_kwargs={ | |
| "primary_engine": primary_engine, | |
| "secondary_engine": secondary_engine, | |
| "primary_engine_kwargs": primary_engine_kwargs or {}, | |
| "secondary_engine_kwargs": secondary_engine_kwargs or {}, | |
| "script": script, | |
| "primary_script_threshold": selected.threshold, | |
| "routing_granularity": routing_granularity, | |
| "secondary_structure_backfill": secondary_structure_backfill, | |
| }, | |
| provenance={ | |
| "calibration": { | |
| "experiment_report": {"path": str(Path(experiment_report)), "sha256": _maybe_sha256(Path(experiment_report))}, | |
| "benchmark_report": {"path": experiment.benchmark_report, "sha256": _maybe_sha256(Path(experiment.benchmark_report))}, | |
| "eval_manifest": {"path": experiment.eval_manifest, "sha256": _maybe_sha256(Path(experiment.eval_manifest))}, | |
| "primary_baseline": primary_baseline, | |
| "secondary_baseline": secondary_baseline, | |
| "metric": metric, | |
| "direction": resolved_direction, | |
| "script": script, | |
| "selected_threshold": selected.threshold, | |
| "threshold_start": threshold_start, | |
| "threshold_end": threshold_end, | |
| "threshold_step": threshold_step, | |
| "routing_granularity": routing_granularity, | |
| "secondary_structure_backfill": secondary_structure_backfill, | |
| } | |
| }, | |
| metrics_report=out / "calibration.json", | |
| notes=notes, | |
| ) | |
| return calibration, model_path | |
| def _load_experiment_report(path: str | Path) -> ExperimentReport: | |
| raw = json.loads(Path(path).read_text(encoding="utf-8")) | |
| if not isinstance(raw, dict): | |
| raise BenchmarkError("experiment report must be a JSON object") | |
| return ExperimentReport( | |
| experiment_id=str(raw.get("experiment_id") or ""), | |
| output_dir=str(raw.get("output_dir") or ""), | |
| eval_manifest=str(raw.get("eval_manifest") or ""), | |
| references_dir=str(raw["references_dir"]) if raw.get("references_dir") is not None else None, | |
| baselines=[str(item) for item in raw.get("baselines", [])], | |
| input_count=int(raw.get("input_count") or 0), | |
| datasets=[str(item) for item in raw.get("datasets", [])], | |
| slice_counts={str(key): int(value) for key, value in dict(raw.get("slice_counts") or {}).items()}, | |
| benchmark_report=str(raw.get("benchmark_report") or ""), | |
| benchmark_status_counts={str(key): int(value) for key, value in dict(raw.get("benchmark_status_counts") or {}).items()}, | |
| validation_report=str(raw["validation_report"]) if raw.get("validation_report") is not None else None, | |
| validation_status=str(raw["validation_status"]) if raw.get("validation_status") is not None else None, | |
| environment=dict(raw.get("environment") or {}), | |
| ) | |
| def _load_benchmark_rows(path: str | Path) -> list[dict[str, Any]]: | |
| payload = json.loads(Path(path).read_text(encoding="utf-8")) | |
| if not isinstance(payload, list): | |
| raise BenchmarkError("benchmark report must be a JSON list") | |
| rows: list[dict[str, Any]] = [] | |
| for index, item in enumerate(payload): | |
| if not isinstance(item, dict): | |
| raise BenchmarkError(f"benchmark row {index} must be an object") | |
| rows.append(dict(item)) | |
| return rows | |
| _DEVANAGARI_RE = re.compile(r"[\u0900-\u097F]") | |
| _LATIN_RE = re.compile(r"[A-Za-z]") | |
| def _script_ratio(text: str, script: str) -> float: | |
| non_space = [char for char in text if not char.isspace()] | |
| if not non_space: | |
| return 0.0 | |
| normalized = script.strip().lower() | |
| if normalized == "devanagari": | |
| matches = sum(1 for char in non_space if _DEVANAGARI_RE.match(char)) | |
| return matches / len(non_space) | |
| if normalized in {"latin", "english"}: | |
| matches = sum(1 for char in non_space if _LATIN_RE.match(char)) | |
| return matches / len(non_space) | |
| raise BenchmarkError(f"unsupported script calibration target: {script!r}") | |
| def _trial_script_thresholds( | |
| experiment: ExperimentReport, | |
| rows: list[dict[str, Any]], | |
| *, | |
| primary_baseline: str, | |
| secondary_baseline: str, | |
| metric: str, | |
| direction: str, | |
| script: str, | |
| threshold_start: float, | |
| threshold_end: float, | |
| threshold_step: float, | |
| ) -> list[ScriptThresholdTrial]: | |
| if threshold_step <= 0: | |
| raise BenchmarkError("threshold_step must be positive") | |
| primary_rows = {sample_key(row): row for row in rows if row.get("baseline") == primary_baseline and row.get("status") == "ok"} | |
| secondary_rows = {sample_key(row): row for row in rows if row.get("baseline") == secondary_baseline and row.get("status") == "ok"} | |
| shared = sorted(set(primary_rows) & set(secondary_rows)) | |
| if not shared: | |
| raise BenchmarkError(f"no paired rows for primary={primary_baseline} vs secondary={secondary_baseline} on metric={metric}") | |
| output_root = Path(experiment.output_dir) / "benchmark" / primary_baseline | |
| thresholds = _threshold_grid(threshold_start, threshold_end, threshold_step) | |
| ratios: dict[str, float] = {} | |
| for sample_id in shared: | |
| input_path = Path(primary_rows[sample_id]["input_path"]) | |
| document_path = output_root / input_path.stem / "document.md" | |
| if not document_path.exists(): | |
| raise BenchmarkError(f"primary benchmark output is missing document.md for {sample_id}: {document_path}") | |
| ratios[sample_id] = _script_ratio(document_path.read_text(encoding="utf-8"), script) | |
| trials: list[ScriptThresholdTrial] = [] | |
| for threshold in thresholds: | |
| chosen_metrics: list[float] = [] | |
| primary_metrics: list[float] = [] | |
| secondary_picks = 0 | |
| for sample_id in shared: | |
| primary_row = primary_rows[sample_id] | |
| secondary_row = secondary_rows[sample_id] | |
| metric_primary = (primary_row.get("metrics") or {}).get(metric) | |
| metric_secondary = (secondary_row.get("metrics") or {}).get(metric) | |
| if not _finite_number(metric_primary) or not _finite_number(metric_secondary): | |
| continue | |
| if ratios[sample_id] > threshold: | |
| chosen_metrics.append(float(metric_secondary)) | |
| secondary_picks += 1 | |
| else: | |
| chosen_metrics.append(float(metric_primary)) | |
| primary_metrics.append(float(metric_primary)) | |
| if not chosen_metrics: | |
| continue | |
| chosen_mean = sum(chosen_metrics) / len(chosen_metrics) | |
| primary_mean = sum(primary_metrics) / len(primary_metrics) | |
| absolute_improvement = _absolute_improvement(chosen_mean, primary_mean, direction) | |
| relative_improvement = _relative_improvement(absolute_improvement, primary_mean) | |
| trials.append( | |
| ScriptThresholdTrial( | |
| threshold=threshold, | |
| pairs=len(chosen_metrics), | |
| secondary_picks=secondary_picks, | |
| chosen_mean=chosen_mean, | |
| primary_mean=primary_mean, | |
| absolute_improvement=absolute_improvement, | |
| relative_improvement=relative_improvement, | |
| ) | |
| ) | |
| return trials | |
| def _trial_thresholds( | |
| rows: list[dict[str, Any]], | |
| *, | |
| candidate_baseline: str, | |
| primary_baseline: str, | |
| metric: str, | |
| direction: str, | |
| threshold_start: float, | |
| threshold_end: float, | |
| threshold_step: float, | |
| ) -> list[ThresholdTrial]: | |
| if threshold_step <= 0: | |
| raise BenchmarkError("threshold_step must be positive") | |
| paired: dict[str, tuple[dict[str, Any], dict[str, Any]]] = {} | |
| candidate_rows = {sample_key(row): row for row in rows if row.get("baseline") == candidate_baseline and row.get("status") == "ok"} | |
| primary_rows = {sample_key(row): row for row in rows if row.get("baseline") == primary_baseline and row.get("status") == "ok"} | |
| for key in sorted(set(candidate_rows) & set(primary_rows)): | |
| candidate_row = candidate_rows[key] | |
| primary_row = primary_rows[key] | |
| if metric not in (candidate_row.get("metrics") or {}) or metric not in (primary_row.get("metrics") or {}): | |
| continue | |
| candidate_quality = (candidate_row.get("metrics") or {}).get("quality_score") | |
| primary_quality = (primary_row.get("metrics") or {}).get("quality_score") | |
| if not _finite_number(candidate_quality) or not _finite_number(primary_quality): | |
| continue | |
| metric_candidate = (candidate_row.get("metrics") or {}).get(metric) | |
| metric_primary = (primary_row.get("metrics") or {}).get(metric) | |
| if not _finite_number(metric_candidate) or not _finite_number(metric_primary): | |
| continue | |
| paired[key] = (candidate_row, primary_row) | |
| if not paired: | |
| raise BenchmarkError(f"no paired rows for candidate={candidate_baseline} vs primary={primary_baseline} on metric={metric}") | |
| thresholds = _threshold_grid(threshold_start, threshold_end, threshold_step) | |
| trials: list[ThresholdTrial] = [] | |
| for threshold in thresholds: | |
| chosen_metrics: list[float] = [] | |
| primary_metrics: list[float] = [] | |
| candidate_picks = 0 | |
| for candidate_row, primary_row in paired.values(): | |
| candidate_quality = float(candidate_row["metrics"]["quality_score"]) | |
| primary_quality = float(primary_row["metrics"]["quality_score"]) | |
| metric_candidate = float(candidate_row["metrics"][metric]) | |
| metric_primary = float(primary_row["metrics"][metric]) | |
| if candidate_quality - primary_quality > threshold: | |
| chosen_metrics.append(metric_candidate) | |
| candidate_picks += 1 | |
| else: | |
| chosen_metrics.append(metric_primary) | |
| primary_metrics.append(metric_primary) | |
| chosen_mean = sum(chosen_metrics) / len(chosen_metrics) | |
| primary_mean = sum(primary_metrics) / len(primary_metrics) | |
| absolute_improvement = _absolute_improvement(chosen_mean, primary_mean, direction) | |
| relative_improvement = _relative_improvement(absolute_improvement, primary_mean) | |
| trials.append( | |
| ThresholdTrial( | |
| threshold=threshold, | |
| pairs=len(chosen_metrics), | |
| candidate_picks=candidate_picks, | |
| chosen_mean=chosen_mean, | |
| primary_mean=primary_mean, | |
| absolute_improvement=absolute_improvement, | |
| relative_improvement=relative_improvement, | |
| ) | |
| ) | |
| return trials | |
| def _threshold_grid(start: float, end: float, step: float) -> list[float]: | |
| values: list[float] = [] | |
| current = start | |
| limit = end + (step / 2) | |
| while current <= limit: | |
| values.append(round(current, 10)) | |
| current += step | |
| return values | |
| def _select_best_trial(trials: list[ThresholdTrial], direction: str) -> ThresholdTrial: | |
| if direction == "lower": | |
| return min(trials, key=lambda item: (item.chosen_mean, item.candidate_picks, -item.threshold)) | |
| return max(trials, key=lambda item: (item.chosen_mean, -item.candidate_picks, item.threshold)) | |
| def _select_best_script_trial(trials: list[ScriptThresholdTrial], direction: str) -> ScriptThresholdTrial: | |
| if direction == "lower": | |
| return min(trials, key=lambda item: (item.chosen_mean, item.secondary_picks, item.threshold)) | |
| return max(trials, key=lambda item: (item.chosen_mean, -item.secondary_picks, -item.threshold)) | |
| def _absolute_improvement(candidate_value: float, baseline_value: float, direction: str) -> float: | |
| if direction == "lower": | |
| return baseline_value - candidate_value | |
| return candidate_value - baseline_value | |
| def _relative_improvement(absolute_improvement: float, baseline_mean: float) -> float: | |
| denominator = abs(baseline_mean) | |
| if denominator == 0: | |
| return math.inf if absolute_improvement > 0 else 0.0 | |
| return absolute_improvement / denominator | |
| def _finite_number(value: object) -> bool: | |
| return isinstance(value, int | float) and math.isfinite(float(value)) | |
| def _maybe_sha256(path: Path) -> str | None: | |
| return sha256_file(path) if path.exists() and path.is_file() else None | |
| def _render_calibration_markdown(report: CalibrationReport) -> str: | |
| lines = [ | |
| "# Quality Router Calibration", | |
| "", | |
| f"Experiment report: `{report.experiment_report}`", | |
| f"Benchmark report: `{report.benchmark_report}`", | |
| f"Eval manifest: `{report.eval_manifest}`", | |
| f"Candidate baseline: `{report.candidate_baseline}`", | |
| f"Primary baseline: `{report.primary_baseline}`", | |
| f"Metric: `{report.metric}`", | |
| f"Direction: `{report.direction}`", | |
| f"Selected threshold: `{report.selected_threshold:.6f}`", | |
| "", | |
| "| threshold | pairs | candidate_picks | chosen_mean | primary_mean | improvement | relative |", | |
| "| ---: | ---: | ---: | ---: | ---: | ---: | ---: |", | |
| ] | |
| for trial in report.trials: | |
| lines.append( | |
| f"| {trial.threshold:.6f} | {trial.pairs} | {trial.candidate_picks} | {trial.chosen_mean:.6f} | {trial.primary_mean:.6f} | {trial.absolute_improvement:.6f} | {trial.relative_improvement:.6f} |" | |
| ) | |
| return "\n".join(lines) + "\n" | |
| def _render_script_calibration_markdown(report: ScriptCalibrationReport) -> str: | |
| lines = [ | |
| "# Script Router Calibration", | |
| "", | |
| f"Experiment report: `{report.experiment_report}`", | |
| f"Benchmark report: `{report.benchmark_report}`", | |
| f"Eval manifest: `{report.eval_manifest}`", | |
| f"Primary baseline: `{report.primary_baseline}`", | |
| f"Secondary baseline: `{report.secondary_baseline}`", | |
| f"Metric: `{report.metric}`", | |
| f"Direction: `{report.direction}`", | |
| f"Script: `{report.script}`", | |
| f"Selected threshold: `{report.selected_threshold:.6f}`", | |
| "", | |
| "| threshold | pairs | secondary_picks | chosen_mean | primary_mean | improvement | relative |", | |
| "| ---: | ---: | ---: | ---: | ---: | ---: | ---: |", | |
| ] | |
| for trial in report.trials: | |
| lines.append( | |
| f"| {trial.threshold:.6f} | {trial.pairs} | {trial.secondary_picks} | {trial.chosen_mean:.6f} | {trial.primary_mean:.6f} | {trial.absolute_improvement:.6f} | {trial.relative_improvement:.6f} |" | |
| ) | |
| return "\n".join(lines) + "\n" | |
| class EnsembleTrial: | |
| biases: dict[str, float] | |
| pairs: int | |
| alternative_picks: int | |
| chosen_mean: float | |
| primary_mean: float | |
| absolute_improvement: float | |
| relative_improvement: float | |
| def to_dict(self) -> dict[str, Any]: | |
| return { | |
| "biases": self.biases, | |
| "pairs": self.pairs, | |
| "alternative_picks": self.alternative_picks, | |
| "chosen_mean": self.chosen_mean, | |
| "primary_mean": self.primary_mean, | |
| "absolute_improvement": self.absolute_improvement, | |
| "relative_improvement": self.relative_improvement, | |
| } | |
| class EnsembleCalibrationReport: | |
| primary_experiment_report: str | |
| variant_experiments: dict[str, str] | |
| benchmark_reports: dict[str, str] | |
| eval_manifest: str | |
| primary_label: str | |
| metric: str | |
| direction: str | |
| selected_biases: dict[str, float] | |
| selected_trial: EnsembleTrial | |
| trials: list[EnsembleTrial] = field(default_factory=list) | |
| def to_dict(self) -> dict[str, Any]: | |
| return { | |
| "primary_experiment_report": self.primary_experiment_report, | |
| "variant_experiments": self.variant_experiments, | |
| "benchmark_reports": self.benchmark_reports, | |
| "eval_manifest": self.eval_manifest, | |
| "primary_label": self.primary_label, | |
| "metric": self.metric, | |
| "direction": self.direction, | |
| "selected_biases": self.selected_biases, | |
| "selected_trial": self.selected_trial.to_dict(), | |
| "trials": [trial.to_dict() for trial in self.trials], | |
| } | |
| def calibrate_tesseract_psm_ensemble( | |
| primary_experiment_report: str | Path, | |
| output_dir: str | Path, | |
| *, | |
| model_id: str, | |
| variant_experiment_reports: dict[str, str | Path], | |
| metric: str = "cer", | |
| direction: str | None = None, | |
| bias_start: float = -0.12, | |
| bias_end: float = 0.12, | |
| bias_step: float = 0.02, | |
| language: str = "nep+eng", | |
| selection_margin: float = 0.0, | |
| notes: str | None = None, | |
| ) -> tuple[EnsembleCalibrationReport, Path]: | |
| if not variant_experiment_reports: | |
| raise BenchmarkError("tesseract ensemble calibration requires at least one variant experiment report") | |
| primary_experiment = _load_experiment_report(primary_experiment_report) | |
| resolved_direction = direction or metric_direction(metric) | |
| primary_rows = _load_variant_rows(primary_experiment.benchmark_report, baseline="tesseract") | |
| variant_rows: dict[str, dict[str, dict[str, Any]]] = {} | |
| variant_reports: dict[str, str] = {} | |
| for label, path in sorted(variant_experiment_reports.items()): | |
| experiment = _load_experiment_report(path) | |
| if experiment.eval_manifest != primary_experiment.eval_manifest: | |
| raise BenchmarkError(f"variant experiment {path} does not use the same eval manifest as the primary experiment") | |
| variant_rows[label] = _load_variant_rows(experiment.benchmark_report, baseline="candidate") | |
| variant_reports[label] = experiment.benchmark_report | |
| shared = set(primary_rows) | |
| for values in variant_rows.values(): | |
| shared &= set(values) | |
| if not shared: | |
| raise BenchmarkError("no shared samples across primary and variant experiment reports") | |
| shared_ids = sorted(shared) | |
| bias_values = _threshold_grid(bias_start, bias_end, bias_step) | |
| trials: list[EnsembleTrial] = [] | |
| labels = sorted(variant_rows) | |
| import itertools | |
| for combo in itertools.product(bias_values, repeat=len(labels)): | |
| biases = dict(zip(labels, combo, strict=True)) | |
| chosen_metrics: list[float] = [] | |
| primary_metrics: list[float] = [] | |
| alternative_picks = 0 | |
| for sample_id in shared_ids: | |
| primary = primary_rows[sample_id] | |
| scored = [(primary["metrics"].get("quality_score", -1.0), primary, None)] | |
| for label in labels: | |
| row = variant_rows[label][sample_id] | |
| scored.append((float(row["metrics"].get("quality_score", -1.0)) + biases[label], row, label)) | |
| selected_quality, selected_row, selected_label = max(scored, key=lambda item: item[0]) | |
| if selected_label is not None and selected_quality <= float(primary["metrics"].get("quality_score", -1.0)) + selection_margin: | |
| selected_row = primary | |
| selected_label = None | |
| chosen_metrics.append(float(selected_row["metrics"][metric])) | |
| primary_metrics.append(float(primary["metrics"][metric])) | |
| if selected_label is not None: | |
| alternative_picks += 1 | |
| chosen_mean = sum(chosen_metrics) / len(chosen_metrics) | |
| primary_mean = sum(primary_metrics) / len(primary_metrics) | |
| absolute_improvement = _absolute_improvement(chosen_mean, primary_mean, resolved_direction) | |
| relative_improvement = _relative_improvement(absolute_improvement, primary_mean) | |
| trials.append( | |
| EnsembleTrial( | |
| biases=biases, | |
| pairs=len(shared_ids), | |
| alternative_picks=alternative_picks, | |
| chosen_mean=chosen_mean, | |
| primary_mean=primary_mean, | |
| absolute_improvement=absolute_improvement, | |
| relative_improvement=relative_improvement, | |
| ) | |
| ) | |
| if not trials: | |
| raise BenchmarkError("tesseract ensemble calibration produced no trials") | |
| selected = _select_best_ensemble_trial(trials, resolved_direction) | |
| report = EnsembleCalibrationReport( | |
| primary_experiment_report=str(Path(primary_experiment_report)), | |
| variant_experiments={label: str(path) for label, path in variant_experiment_reports.items()}, | |
| benchmark_reports={"primary": primary_experiment.benchmark_report, **variant_reports}, | |
| eval_manifest=primary_experiment.eval_manifest, | |
| primary_label="psm6", | |
| metric=metric, | |
| direction=resolved_direction, | |
| selected_biases=selected.biases, | |
| selected_trial=selected, | |
| trials=trials, | |
| ) | |
| out = Path(output_dir) | |
| out.mkdir(parents=True, exist_ok=True) | |
| (out / "calibration.json").write_text(json.dumps(report.to_dict(), ensure_ascii=False, indent=2) + "\n", encoding="utf-8") | |
| (out / "calibration.md").write_text(_render_ensemble_calibration_markdown(report), encoding="utf-8") | |
| alternatives = [ | |
| {"label": label, "engine": "tesseract", "engine_kwargs": {"language": language, "psm": int(label.removeprefix("psm"))}, "quality_bias": bias} | |
| for label, bias in sorted(selected.biases.items()) | |
| ] | |
| model_path = write_model_card( | |
| out / "model-card.json", | |
| model_id=model_id, | |
| backend="quality_ranked_ensemble", | |
| base_model="tesseract-ocr", | |
| backend_kwargs={ | |
| "primary_engine": "tesseract", | |
| "primary_engine_kwargs": {"language": language, "psm": 6}, | |
| "alternative_engines": alternatives, | |
| "selection_margin": selection_margin, | |
| }, | |
| provenance={ | |
| "calibration": { | |
| "eval_manifest": {"path": primary_experiment.eval_manifest, "sha256": _maybe_sha256(Path(primary_experiment.eval_manifest))}, | |
| "primary_experiment_report": {"path": str(Path(primary_experiment_report)), "sha256": _maybe_sha256(Path(primary_experiment_report))}, | |
| "variant_experiments": { | |
| label: {"path": str(path), "sha256": _maybe_sha256(Path(path))} | |
| for label, path in variant_experiment_reports.items() | |
| }, | |
| "metric": metric, | |
| "direction": resolved_direction, | |
| "selected_biases": selected.biases, | |
| "selection_margin": selection_margin, | |
| "bias_start": bias_start, | |
| "bias_end": bias_end, | |
| "bias_step": bias_step, | |
| } | |
| }, | |
| metrics_report=out / "calibration.json", | |
| notes=notes, | |
| ) | |
| return report, model_path | |
| def _load_variant_rows(path: str | Path, *, baseline: str) -> dict[str, dict[str, Any]]: | |
| rows = _load_benchmark_rows(path) | |
| selected: dict[str, dict[str, Any]] = {} | |
| for row in rows: | |
| if row.get("baseline") != baseline or row.get("status") != "ok": | |
| continue | |
| selected[sample_key(row)] = row | |
| if not selected: | |
| raise BenchmarkError(f"no benchmark rows found for baseline={baseline} in {path}") | |
| return selected | |
| def _select_best_ensemble_trial(trials: list[EnsembleTrial], direction: str) -> EnsembleTrial: | |
| if direction == "lower": | |
| return min(trials, key=lambda item: (item.chosen_mean, item.alternative_picks, sorted(item.biases.items()))) | |
| return max(trials, key=lambda item: (item.chosen_mean, -item.alternative_picks, sorted(item.biases.items()))) | |
| def _render_ensemble_calibration_markdown(report: EnsembleCalibrationReport) -> str: | |
| labels = sorted(report.selected_biases) | |
| lines = [ | |
| "# Quality Ranked Ensemble Calibration", | |
| "", | |
| f"Primary experiment report: `{report.primary_experiment_report}`", | |
| f"Eval manifest: `{report.eval_manifest}`", | |
| f"Metric: `{report.metric}`", | |
| f"Direction: `{report.direction}`", | |
| f"Selected biases: `{report.selected_biases}`", | |
| "", | |
| "| biases | pairs | alternative_picks | chosen_mean | primary_mean | improvement | relative |", | |
| "| --- | ---: | ---: | ---: | ---: | ---: | ---: |", | |
| ] | |
| for trial in report.trials: | |
| bias_text = ", ".join(f"{label}={trial.biases[label]:.2f}" for label in labels) | |
| lines.append( | |
| f"| {bias_text} | {trial.pairs} | {trial.alternative_picks} | {trial.chosen_mean:.6f} | {trial.primary_mean:.6f} | {trial.absolute_improvement:.6f} | {trial.relative_improvement:.6f} |" | |
| ) | |
| return "\n".join(lines) + "\n" | |