prosperious's picture
Add 1000 files (OCR_INDRA/.gitignore ...)
f9bc0c1 verified
Raw History Blame Contribute Delete
34.1 kB
"""Calibration helpers for routed OCR candidates."""
from __future__ import annotations
import json
import math
import re
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
from .errors import BenchmarkError
from .experiments import ExperimentReport
from .manifest import sha256_file
from .models import write_model_card
from .validation import metric_direction, sample_key
@dataclass(slots=True)
class ThresholdTrial:
threshold: float
pairs: int
candidate_picks: int
chosen_mean: float
primary_mean: float
absolute_improvement: float
relative_improvement: float
def to_dict(self) -> dict[str, Any]:
return {
"threshold": self.threshold,
"pairs": self.pairs,
"candidate_picks": self.candidate_picks,
"chosen_mean": self.chosen_mean,
"primary_mean": self.primary_mean,
"absolute_improvement": self.absolute_improvement,
"relative_improvement": self.relative_improvement,
}
@dataclass(slots=True)
class CalibrationReport:
experiment_report: str
benchmark_report: str
eval_manifest: str
candidate_baseline: str
primary_baseline: str
metric: str
direction: str
selected_threshold: float
selected_trial: ThresholdTrial
trials: list[ThresholdTrial] = field(default_factory=list)
def to_dict(self) -> dict[str, Any]:
return {
"experiment_report": self.experiment_report,
"benchmark_report": self.benchmark_report,
"eval_manifest": self.eval_manifest,
"candidate_baseline": self.candidate_baseline,
"primary_baseline": self.primary_baseline,
"metric": self.metric,
"direction": self.direction,
"selected_threshold": self.selected_threshold,
"selected_trial": self.selected_trial.to_dict(),
"trials": [trial.to_dict() for trial in self.trials],
}
@dataclass(slots=True)
class ScriptThresholdTrial:
threshold: float
pairs: int
secondary_picks: int
chosen_mean: float
primary_mean: float
absolute_improvement: float
relative_improvement: float
def to_dict(self) -> dict[str, Any]:
return {
"threshold": self.threshold,
"pairs": self.pairs,
"secondary_picks": self.secondary_picks,
"chosen_mean": self.chosen_mean,
"primary_mean": self.primary_mean,
"absolute_improvement": self.absolute_improvement,
"relative_improvement": self.relative_improvement,
}
@dataclass(slots=True)
class ScriptCalibrationReport:
experiment_report: str
benchmark_report: str
eval_manifest: str
primary_baseline: str
secondary_baseline: str
metric: str
direction: str
script: str
selected_threshold: float
selected_trial: ScriptThresholdTrial
trials: list[ScriptThresholdTrial] = field(default_factory=list)
def to_dict(self) -> dict[str, Any]:
return {
"experiment_report": self.experiment_report,
"benchmark_report": self.benchmark_report,
"eval_manifest": self.eval_manifest,
"primary_baseline": self.primary_baseline,
"secondary_baseline": self.secondary_baseline,
"metric": self.metric,
"direction": self.direction,
"script": self.script,
"selected_threshold": self.selected_threshold,
"selected_trial": self.selected_trial.to_dict(),
"trials": [trial.to_dict() for trial in self.trials],
}
def calibrate_quality_router(
experiment_report: str | Path,
output_dir: str | Path,
*,
model_id: str,
primary_engine: str,
secondary_engine: str,
primary_engine_kwargs: dict[str, Any] | None = None,
secondary_engine_kwargs: dict[str, Any] | None = None,
candidate_baseline: str = "candidate",
primary_baseline: str = "tesseract",
metric: str = "cer",
direction: str | None = None,
threshold_start: float = -0.20,
threshold_end: float = 0.20,
threshold_step: float = 0.01,
base_model: str = "tesseract-ocr",
notes: str | None = None,
) -> tuple[CalibrationReport, Path]:
experiment = _load_experiment_report(experiment_report)
report_rows = _load_benchmark_rows(experiment.benchmark_report)
resolved_direction = direction or metric_direction(metric)
trials = _trial_thresholds(
report_rows,
candidate_baseline=candidate_baseline,
primary_baseline=primary_baseline,
metric=metric,
direction=resolved_direction,
threshold_start=threshold_start,
threshold_end=threshold_end,
threshold_step=threshold_step,
)
if not trials:
raise BenchmarkError("quality router calibration produced no valid threshold trials")
selected = _select_best_trial(trials, resolved_direction)
calibration = CalibrationReport(
experiment_report=str(Path(experiment_report)),
benchmark_report=experiment.benchmark_report,
eval_manifest=experiment.eval_manifest,
candidate_baseline=candidate_baseline,
primary_baseline=primary_baseline,
metric=metric,
direction=resolved_direction,
selected_threshold=selected.threshold,
selected_trial=selected,
trials=trials,
)
out = Path(output_dir)
out.mkdir(parents=True, exist_ok=True)
(out / "calibration.json").write_text(json.dumps(calibration.to_dict(), ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
(out / "calibration.md").write_text(_render_calibration_markdown(calibration), encoding="utf-8")
model_path = write_model_card(
out / "model-card.json",
model_id=model_id,
backend="quality_select_composite",
base_model=base_model,
backend_kwargs={
"primary_engine": primary_engine,
"secondary_engine": secondary_engine,
"primary_engine_kwargs": primary_engine_kwargs or {},
"secondary_engine_kwargs": secondary_engine_kwargs or {},
"secondary_quality_margin": selected.threshold,
},
provenance={
"calibration": {
"experiment_report": {"path": str(Path(experiment_report)), "sha256": _maybe_sha256(Path(experiment_report))},
"benchmark_report": {"path": experiment.benchmark_report, "sha256": _maybe_sha256(Path(experiment.benchmark_report))},
"eval_manifest": {"path": experiment.eval_manifest, "sha256": _maybe_sha256(Path(experiment.eval_manifest))},
"candidate_baseline": candidate_baseline,
"primary_baseline": primary_baseline,
"metric": metric,
"direction": resolved_direction,
"selected_threshold": selected.threshold,
"threshold_start": threshold_start,
"threshold_end": threshold_end,
"threshold_step": threshold_step,
}
},
metrics_report=out / "calibration.json",
notes=notes,
)
return calibration, model_path
def calibrate_script_router(
experiment_report: str | Path,
output_dir: str | Path,
*,
model_id: str,
primary_engine: str,
secondary_engine: str,
primary_engine_kwargs: dict[str, Any] | None = None,
secondary_engine_kwargs: dict[str, Any] | None = None,
primary_baseline: str = "tesseract",
secondary_baseline: str = "surya",
metric: str = "cer",
direction: str | None = None,
script: str = "devanagari",
threshold_start: float = 0.0,
threshold_end: float = 0.6,
threshold_step: float = 0.05,
routing_granularity: str = "document",
secondary_structure_backfill: bool = False,
base_model: str = "tesseract+surya",
notes: str | None = None,
) -> tuple[ScriptCalibrationReport, Path]:
experiment = _load_experiment_report(experiment_report)
report_rows = _load_benchmark_rows(experiment.benchmark_report)
resolved_direction = direction or metric_direction(metric)
trials = _trial_script_thresholds(
experiment,
report_rows,
primary_baseline=primary_baseline,
secondary_baseline=secondary_baseline,
metric=metric,
direction=resolved_direction,
script=script,
threshold_start=threshold_start,
threshold_end=threshold_end,
threshold_step=threshold_step,
)
if not trials:
raise BenchmarkError("script router calibration produced no valid threshold trials")
selected = _select_best_script_trial(trials, resolved_direction)
calibration = ScriptCalibrationReport(
experiment_report=str(Path(experiment_report)),
benchmark_report=experiment.benchmark_report,
eval_manifest=experiment.eval_manifest,
primary_baseline=primary_baseline,
secondary_baseline=secondary_baseline,
metric=metric,
direction=resolved_direction,
script=script,
selected_threshold=selected.threshold,
selected_trial=selected,
trials=trials,
)
out = Path(output_dir)
out.mkdir(parents=True, exist_ok=True)
(out / "calibration.json").write_text(json.dumps(calibration.to_dict(), ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
(out / "calibration.md").write_text(_render_script_calibration_markdown(calibration), encoding="utf-8")
model_path = write_model_card(
out / "model-card.json",
model_id=model_id,
backend="script_select_composite",
base_model=base_model,
backend_kwargs={
"primary_engine": primary_engine,
"secondary_engine": secondary_engine,
"primary_engine_kwargs": primary_engine_kwargs or {},
"secondary_engine_kwargs": secondary_engine_kwargs or {},
"script": script,
"primary_script_threshold": selected.threshold,
"routing_granularity": routing_granularity,
"secondary_structure_backfill": secondary_structure_backfill,
},
provenance={
"calibration": {
"experiment_report": {"path": str(Path(experiment_report)), "sha256": _maybe_sha256(Path(experiment_report))},
"benchmark_report": {"path": experiment.benchmark_report, "sha256": _maybe_sha256(Path(experiment.benchmark_report))},
"eval_manifest": {"path": experiment.eval_manifest, "sha256": _maybe_sha256(Path(experiment.eval_manifest))},
"primary_baseline": primary_baseline,
"secondary_baseline": secondary_baseline,
"metric": metric,
"direction": resolved_direction,
"script": script,
"selected_threshold": selected.threshold,
"threshold_start": threshold_start,
"threshold_end": threshold_end,
"threshold_step": threshold_step,
"routing_granularity": routing_granularity,
"secondary_structure_backfill": secondary_structure_backfill,
}
},
metrics_report=out / "calibration.json",
notes=notes,
)
return calibration, model_path
def _load_experiment_report(path: str | Path) -> ExperimentReport:
raw = json.loads(Path(path).read_text(encoding="utf-8"))
if not isinstance(raw, dict):
raise BenchmarkError("experiment report must be a JSON object")
return ExperimentReport(
experiment_id=str(raw.get("experiment_id") or ""),
output_dir=str(raw.get("output_dir") or ""),
eval_manifest=str(raw.get("eval_manifest") or ""),
references_dir=str(raw["references_dir"]) if raw.get("references_dir") is not None else None,
baselines=[str(item) for item in raw.get("baselines", [])],
input_count=int(raw.get("input_count") or 0),
datasets=[str(item) for item in raw.get("datasets", [])],
slice_counts={str(key): int(value) for key, value in dict(raw.get("slice_counts") or {}).items()},
benchmark_report=str(raw.get("benchmark_report") or ""),
benchmark_status_counts={str(key): int(value) for key, value in dict(raw.get("benchmark_status_counts") or {}).items()},
validation_report=str(raw["validation_report"]) if raw.get("validation_report") is not None else None,
validation_status=str(raw["validation_status"]) if raw.get("validation_status") is not None else None,
environment=dict(raw.get("environment") or {}),
)
def _load_benchmark_rows(path: str | Path) -> list[dict[str, Any]]:
payload = json.loads(Path(path).read_text(encoding="utf-8"))
if not isinstance(payload, list):
raise BenchmarkError("benchmark report must be a JSON list")
rows: list[dict[str, Any]] = []
for index, item in enumerate(payload):
if not isinstance(item, dict):
raise BenchmarkError(f"benchmark row {index} must be an object")
rows.append(dict(item))
return rows
_DEVANAGARI_RE = re.compile(r"[\u0900-\u097F]")
_LATIN_RE = re.compile(r"[A-Za-z]")
def _script_ratio(text: str, script: str) -> float:
non_space = [char for char in text if not char.isspace()]
if not non_space:
return 0.0
normalized = script.strip().lower()
if normalized == "devanagari":
matches = sum(1 for char in non_space if _DEVANAGARI_RE.match(char))
return matches / len(non_space)
if normalized in {"latin", "english"}:
matches = sum(1 for char in non_space if _LATIN_RE.match(char))
return matches / len(non_space)
raise BenchmarkError(f"unsupported script calibration target: {script!r}")
def _trial_script_thresholds(
experiment: ExperimentReport,
rows: list[dict[str, Any]],
*,
primary_baseline: str,
secondary_baseline: str,
metric: str,
direction: str,
script: str,
threshold_start: float,
threshold_end: float,
threshold_step: float,
) -> list[ScriptThresholdTrial]:
if threshold_step <= 0:
raise BenchmarkError("threshold_step must be positive")
primary_rows = {sample_key(row): row for row in rows if row.get("baseline") == primary_baseline and row.get("status") == "ok"}
secondary_rows = {sample_key(row): row for row in rows if row.get("baseline") == secondary_baseline and row.get("status") == "ok"}
shared = sorted(set(primary_rows) & set(secondary_rows))
if not shared:
raise BenchmarkError(f"no paired rows for primary={primary_baseline} vs secondary={secondary_baseline} on metric={metric}")
output_root = Path(experiment.output_dir) / "benchmark" / primary_baseline
thresholds = _threshold_grid(threshold_start, threshold_end, threshold_step)
ratios: dict[str, float] = {}
for sample_id in shared:
input_path = Path(primary_rows[sample_id]["input_path"])
document_path = output_root / input_path.stem / "document.md"
if not document_path.exists():
raise BenchmarkError(f"primary benchmark output is missing document.md for {sample_id}: {document_path}")
ratios[sample_id] = _script_ratio(document_path.read_text(encoding="utf-8"), script)
trials: list[ScriptThresholdTrial] = []
for threshold in thresholds:
chosen_metrics: list[float] = []
primary_metrics: list[float] = []
secondary_picks = 0
for sample_id in shared:
primary_row = primary_rows[sample_id]
secondary_row = secondary_rows[sample_id]
metric_primary = (primary_row.get("metrics") or {}).get(metric)
metric_secondary = (secondary_row.get("metrics") or {}).get(metric)
if not _finite_number(metric_primary) or not _finite_number(metric_secondary):
continue
if ratios[sample_id] > threshold:
chosen_metrics.append(float(metric_secondary))
secondary_picks += 1
else:
chosen_metrics.append(float(metric_primary))
primary_metrics.append(float(metric_primary))
if not chosen_metrics:
continue
chosen_mean = sum(chosen_metrics) / len(chosen_metrics)
primary_mean = sum(primary_metrics) / len(primary_metrics)
absolute_improvement = _absolute_improvement(chosen_mean, primary_mean, direction)
relative_improvement = _relative_improvement(absolute_improvement, primary_mean)
trials.append(
ScriptThresholdTrial(
threshold=threshold,
pairs=len(chosen_metrics),
secondary_picks=secondary_picks,
chosen_mean=chosen_mean,
primary_mean=primary_mean,
absolute_improvement=absolute_improvement,
relative_improvement=relative_improvement,
)
)
return trials
def _trial_thresholds(
rows: list[dict[str, Any]],
*,
candidate_baseline: str,
primary_baseline: str,
metric: str,
direction: str,
threshold_start: float,
threshold_end: float,
threshold_step: float,
) -> list[ThresholdTrial]:
if threshold_step <= 0:
raise BenchmarkError("threshold_step must be positive")
paired: dict[str, tuple[dict[str, Any], dict[str, Any]]] = {}
candidate_rows = {sample_key(row): row for row in rows if row.get("baseline") == candidate_baseline and row.get("status") == "ok"}
primary_rows = {sample_key(row): row for row in rows if row.get("baseline") == primary_baseline and row.get("status") == "ok"}
for key in sorted(set(candidate_rows) & set(primary_rows)):
candidate_row = candidate_rows[key]
primary_row = primary_rows[key]
if metric not in (candidate_row.get("metrics") or {}) or metric not in (primary_row.get("metrics") or {}):
continue
candidate_quality = (candidate_row.get("metrics") or {}).get("quality_score")
primary_quality = (primary_row.get("metrics") or {}).get("quality_score")
if not _finite_number(candidate_quality) or not _finite_number(primary_quality):
continue
metric_candidate = (candidate_row.get("metrics") or {}).get(metric)
metric_primary = (primary_row.get("metrics") or {}).get(metric)
if not _finite_number(metric_candidate) or not _finite_number(metric_primary):
continue
paired[key] = (candidate_row, primary_row)
if not paired:
raise BenchmarkError(f"no paired rows for candidate={candidate_baseline} vs primary={primary_baseline} on metric={metric}")
thresholds = _threshold_grid(threshold_start, threshold_end, threshold_step)
trials: list[ThresholdTrial] = []
for threshold in thresholds:
chosen_metrics: list[float] = []
primary_metrics: list[float] = []
candidate_picks = 0
for candidate_row, primary_row in paired.values():
candidate_quality = float(candidate_row["metrics"]["quality_score"])
primary_quality = float(primary_row["metrics"]["quality_score"])
metric_candidate = float(candidate_row["metrics"][metric])
metric_primary = float(primary_row["metrics"][metric])
if candidate_quality - primary_quality > threshold:
chosen_metrics.append(metric_candidate)
candidate_picks += 1
else:
chosen_metrics.append(metric_primary)
primary_metrics.append(metric_primary)
chosen_mean = sum(chosen_metrics) / len(chosen_metrics)
primary_mean = sum(primary_metrics) / len(primary_metrics)
absolute_improvement = _absolute_improvement(chosen_mean, primary_mean, direction)
relative_improvement = _relative_improvement(absolute_improvement, primary_mean)
trials.append(
ThresholdTrial(
threshold=threshold,
pairs=len(chosen_metrics),
candidate_picks=candidate_picks,
chosen_mean=chosen_mean,
primary_mean=primary_mean,
absolute_improvement=absolute_improvement,
relative_improvement=relative_improvement,
)
)
return trials
def _threshold_grid(start: float, end: float, step: float) -> list[float]:
values: list[float] = []
current = start
limit = end + (step / 2)
while current <= limit:
values.append(round(current, 10))
current += step
return values
def _select_best_trial(trials: list[ThresholdTrial], direction: str) -> ThresholdTrial:
if direction == "lower":
return min(trials, key=lambda item: (item.chosen_mean, item.candidate_picks, -item.threshold))
return max(trials, key=lambda item: (item.chosen_mean, -item.candidate_picks, item.threshold))
def _select_best_script_trial(trials: list[ScriptThresholdTrial], direction: str) -> ScriptThresholdTrial:
if direction == "lower":
return min(trials, key=lambda item: (item.chosen_mean, item.secondary_picks, item.threshold))
return max(trials, key=lambda item: (item.chosen_mean, -item.secondary_picks, -item.threshold))
def _absolute_improvement(candidate_value: float, baseline_value: float, direction: str) -> float:
if direction == "lower":
return baseline_value - candidate_value
return candidate_value - baseline_value
def _relative_improvement(absolute_improvement: float, baseline_mean: float) -> float:
denominator = abs(baseline_mean)
if denominator == 0:
return math.inf if absolute_improvement > 0 else 0.0
return absolute_improvement / denominator
def _finite_number(value: object) -> bool:
return isinstance(value, int | float) and math.isfinite(float(value))
def _maybe_sha256(path: Path) -> str | None:
return sha256_file(path) if path.exists() and path.is_file() else None
def _render_calibration_markdown(report: CalibrationReport) -> str:
lines = [
"# Quality Router Calibration",
"",
f"Experiment report: `{report.experiment_report}`",
f"Benchmark report: `{report.benchmark_report}`",
f"Eval manifest: `{report.eval_manifest}`",
f"Candidate baseline: `{report.candidate_baseline}`",
f"Primary baseline: `{report.primary_baseline}`",
f"Metric: `{report.metric}`",
f"Direction: `{report.direction}`",
f"Selected threshold: `{report.selected_threshold:.6f}`",
"",
"| threshold | pairs | candidate_picks | chosen_mean | primary_mean | improvement | relative |",
"| ---: | ---: | ---: | ---: | ---: | ---: | ---: |",
]
for trial in report.trials:
lines.append(
f"| {trial.threshold:.6f} | {trial.pairs} | {trial.candidate_picks} | {trial.chosen_mean:.6f} | {trial.primary_mean:.6f} | {trial.absolute_improvement:.6f} | {trial.relative_improvement:.6f} |"
)
return "\n".join(lines) + "\n"
def _render_script_calibration_markdown(report: ScriptCalibrationReport) -> str:
lines = [
"# Script Router Calibration",
"",
f"Experiment report: `{report.experiment_report}`",
f"Benchmark report: `{report.benchmark_report}`",
f"Eval manifest: `{report.eval_manifest}`",
f"Primary baseline: `{report.primary_baseline}`",
f"Secondary baseline: `{report.secondary_baseline}`",
f"Metric: `{report.metric}`",
f"Direction: `{report.direction}`",
f"Script: `{report.script}`",
f"Selected threshold: `{report.selected_threshold:.6f}`",
"",
"| threshold | pairs | secondary_picks | chosen_mean | primary_mean | improvement | relative |",
"| ---: | ---: | ---: | ---: | ---: | ---: | ---: |",
]
for trial in report.trials:
lines.append(
f"| {trial.threshold:.6f} | {trial.pairs} | {trial.secondary_picks} | {trial.chosen_mean:.6f} | {trial.primary_mean:.6f} | {trial.absolute_improvement:.6f} | {trial.relative_improvement:.6f} |"
)
return "\n".join(lines) + "\n"
@dataclass(slots=True)
class EnsembleTrial:
biases: dict[str, float]
pairs: int
alternative_picks: int
chosen_mean: float
primary_mean: float
absolute_improvement: float
relative_improvement: float
def to_dict(self) -> dict[str, Any]:
return {
"biases": self.biases,
"pairs": self.pairs,
"alternative_picks": self.alternative_picks,
"chosen_mean": self.chosen_mean,
"primary_mean": self.primary_mean,
"absolute_improvement": self.absolute_improvement,
"relative_improvement": self.relative_improvement,
}
@dataclass(slots=True)
class EnsembleCalibrationReport:
primary_experiment_report: str
variant_experiments: dict[str, str]
benchmark_reports: dict[str, str]
eval_manifest: str
primary_label: str
metric: str
direction: str
selected_biases: dict[str, float]
selected_trial: EnsembleTrial
trials: list[EnsembleTrial] = field(default_factory=list)
def to_dict(self) -> dict[str, Any]:
return {
"primary_experiment_report": self.primary_experiment_report,
"variant_experiments": self.variant_experiments,
"benchmark_reports": self.benchmark_reports,
"eval_manifest": self.eval_manifest,
"primary_label": self.primary_label,
"metric": self.metric,
"direction": self.direction,
"selected_biases": self.selected_biases,
"selected_trial": self.selected_trial.to_dict(),
"trials": [trial.to_dict() for trial in self.trials],
}
def calibrate_tesseract_psm_ensemble(
primary_experiment_report: str | Path,
output_dir: str | Path,
*,
model_id: str,
variant_experiment_reports: dict[str, str | Path],
metric: str = "cer",
direction: str | None = None,
bias_start: float = -0.12,
bias_end: float = 0.12,
bias_step: float = 0.02,
language: str = "nep+eng",
selection_margin: float = 0.0,
notes: str | None = None,
) -> tuple[EnsembleCalibrationReport, Path]:
if not variant_experiment_reports:
raise BenchmarkError("tesseract ensemble calibration requires at least one variant experiment report")
primary_experiment = _load_experiment_report(primary_experiment_report)
resolved_direction = direction or metric_direction(metric)
primary_rows = _load_variant_rows(primary_experiment.benchmark_report, baseline="tesseract")
variant_rows: dict[str, dict[str, dict[str, Any]]] = {}
variant_reports: dict[str, str] = {}
for label, path in sorted(variant_experiment_reports.items()):
experiment = _load_experiment_report(path)
if experiment.eval_manifest != primary_experiment.eval_manifest:
raise BenchmarkError(f"variant experiment {path} does not use the same eval manifest as the primary experiment")
variant_rows[label] = _load_variant_rows(experiment.benchmark_report, baseline="candidate")
variant_reports[label] = experiment.benchmark_report
shared = set(primary_rows)
for values in variant_rows.values():
shared &= set(values)
if not shared:
raise BenchmarkError("no shared samples across primary and variant experiment reports")
shared_ids = sorted(shared)
bias_values = _threshold_grid(bias_start, bias_end, bias_step)
trials: list[EnsembleTrial] = []
labels = sorted(variant_rows)
import itertools
for combo in itertools.product(bias_values, repeat=len(labels)):
biases = dict(zip(labels, combo, strict=True))
chosen_metrics: list[float] = []
primary_metrics: list[float] = []
alternative_picks = 0
for sample_id in shared_ids:
primary = primary_rows[sample_id]
scored = [(primary["metrics"].get("quality_score", -1.0), primary, None)]
for label in labels:
row = variant_rows[label][sample_id]
scored.append((float(row["metrics"].get("quality_score", -1.0)) + biases[label], row, label))
selected_quality, selected_row, selected_label = max(scored, key=lambda item: item[0])
if selected_label is not None and selected_quality <= float(primary["metrics"].get("quality_score", -1.0)) + selection_margin:
selected_row = primary
selected_label = None
chosen_metrics.append(float(selected_row["metrics"][metric]))
primary_metrics.append(float(primary["metrics"][metric]))
if selected_label is not None:
alternative_picks += 1
chosen_mean = sum(chosen_metrics) / len(chosen_metrics)
primary_mean = sum(primary_metrics) / len(primary_metrics)
absolute_improvement = _absolute_improvement(chosen_mean, primary_mean, resolved_direction)
relative_improvement = _relative_improvement(absolute_improvement, primary_mean)
trials.append(
EnsembleTrial(
biases=biases,
pairs=len(shared_ids),
alternative_picks=alternative_picks,
chosen_mean=chosen_mean,
primary_mean=primary_mean,
absolute_improvement=absolute_improvement,
relative_improvement=relative_improvement,
)
)
if not trials:
raise BenchmarkError("tesseract ensemble calibration produced no trials")
selected = _select_best_ensemble_trial(trials, resolved_direction)
report = EnsembleCalibrationReport(
primary_experiment_report=str(Path(primary_experiment_report)),
variant_experiments={label: str(path) for label, path in variant_experiment_reports.items()},
benchmark_reports={"primary": primary_experiment.benchmark_report, **variant_reports},
eval_manifest=primary_experiment.eval_manifest,
primary_label="psm6",
metric=metric,
direction=resolved_direction,
selected_biases=selected.biases,
selected_trial=selected,
trials=trials,
)
out = Path(output_dir)
out.mkdir(parents=True, exist_ok=True)
(out / "calibration.json").write_text(json.dumps(report.to_dict(), ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
(out / "calibration.md").write_text(_render_ensemble_calibration_markdown(report), encoding="utf-8")
alternatives = [
{"label": label, "engine": "tesseract", "engine_kwargs": {"language": language, "psm": int(label.removeprefix("psm"))}, "quality_bias": bias}
for label, bias in sorted(selected.biases.items())
]
model_path = write_model_card(
out / "model-card.json",
model_id=model_id,
backend="quality_ranked_ensemble",
base_model="tesseract-ocr",
backend_kwargs={
"primary_engine": "tesseract",
"primary_engine_kwargs": {"language": language, "psm": 6},
"alternative_engines": alternatives,
"selection_margin": selection_margin,
},
provenance={
"calibration": {
"eval_manifest": {"path": primary_experiment.eval_manifest, "sha256": _maybe_sha256(Path(primary_experiment.eval_manifest))},
"primary_experiment_report": {"path": str(Path(primary_experiment_report)), "sha256": _maybe_sha256(Path(primary_experiment_report))},
"variant_experiments": {
label: {"path": str(path), "sha256": _maybe_sha256(Path(path))}
for label, path in variant_experiment_reports.items()
},
"metric": metric,
"direction": resolved_direction,
"selected_biases": selected.biases,
"selection_margin": selection_margin,
"bias_start": bias_start,
"bias_end": bias_end,
"bias_step": bias_step,
}
},
metrics_report=out / "calibration.json",
notes=notes,
)
return report, model_path
def _load_variant_rows(path: str | Path, *, baseline: str) -> dict[str, dict[str, Any]]:
rows = _load_benchmark_rows(path)
selected: dict[str, dict[str, Any]] = {}
for row in rows:
if row.get("baseline") != baseline or row.get("status") != "ok":
continue
selected[sample_key(row)] = row
if not selected:
raise BenchmarkError(f"no benchmark rows found for baseline={baseline} in {path}")
return selected
def _select_best_ensemble_trial(trials: list[EnsembleTrial], direction: str) -> EnsembleTrial:
if direction == "lower":
return min(trials, key=lambda item: (item.chosen_mean, item.alternative_picks, sorted(item.biases.items())))
return max(trials, key=lambda item: (item.chosen_mean, -item.alternative_picks, sorted(item.biases.items())))
def _render_ensemble_calibration_markdown(report: EnsembleCalibrationReport) -> str:
labels = sorted(report.selected_biases)
lines = [
"# Quality Ranked Ensemble Calibration",
"",
f"Primary experiment report: `{report.primary_experiment_report}`",
f"Eval manifest: `{report.eval_manifest}`",
f"Metric: `{report.metric}`",
f"Direction: `{report.direction}`",
f"Selected biases: `{report.selected_biases}`",
"",
"| biases | pairs | alternative_picks | chosen_mean | primary_mean | improvement | relative |",
"| --- | ---: | ---: | ---: | ---: | ---: | ---: |",
]
for trial in report.trials:
bias_text = ", ".join(f"{label}={trial.biases[label]:.2f}" for label in labels)
lines.append(
f"| {bias_text} | {trial.pairs} | {trial.alternative_picks} | {trial.chosen_mean:.6f} | {trial.primary_mean:.6f} | {trial.absolute_improvement:.6f} | {trial.relative_improvement:.6f} |"
)
return "\n".join(lines) + "\n"