prosperious's picture
Add 1000 files (OCR_INDRA/.gitignore ...)
f9bc0c1 verified
Raw History Blame Contribute Delete
11.2 kB
"""Reproducible experiment runner for benchmark and validation jobs."""
from __future__ import annotations
import json
import platform
from dataclasses import dataclass, field
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
from . import __version__
from .benchmark import BenchmarkResult, run_benchmark
from .errors import BenchmarkError
from .manifest import ManifestEntry, load_manifest, sha256_file
from .preflight import run_claim_preflight
from .validation import ValidationReport, validate_claim
@dataclass(slots=True)
class ExperimentReport:
experiment_id: str
output_dir: str
eval_manifest: str
references_dir: str | None
baselines: list[str]
input_count: int
datasets: list[str]
slice_counts: dict[str, int]
benchmark_report: str
benchmark_status_counts: dict[str, int]
preflight_report: str | None = None
preflight_passed: bool | None = None
validation_report: str | None = None
validation_status: str | None = None
environment: dict[str, Any] = field(default_factory=dict)
def to_dict(self) -> dict[str, Any]:
return {
"experiment_id": self.experiment_id,
"output_dir": self.output_dir,
"eval_manifest": self.eval_manifest,
"references_dir": self.references_dir,
"baselines": self.baselines,
"input_count": self.input_count,
"datasets": self.datasets,
"slice_counts": self.slice_counts,
"benchmark_report": self.benchmark_report,
"benchmark_status_counts": self.benchmark_status_counts,
"preflight_report": self.preflight_report,
"preflight_passed": self.preflight_passed,
"validation_report": self.validation_report,
"validation_status": self.validation_status,
"environment": self.environment,
}
def run_experiment(
eval_manifest: str | Path,
output_dir: str | Path,
*,
baselines: list[str],
references_dir: str | Path | None = None,
validation_config: str | Path | None = None,
train_manifests: list[str | Path] | None = None,
candidate_model_config: str | Path | None = None,
fallback_engine: str | None = None,
fallback_model_config: str | Path | None = None,
low_confidence_threshold: float = 0.80,
fallback_min_quality_score: float | None = None,
gorkhapatra_review_audits: list[str | Path] | None = None,
run_validation: bool = True,
run_preflight: bool = False,
require_claim_ready_eval_pack: bool = True,
require_trained_recognizer: bool = False,
require_model_admission: bool = False,
) -> ExperimentReport:
if not baselines:
raise BenchmarkError("run-experiment requires at least one baseline")
eval_path = Path(eval_manifest)
entries = load_manifest(eval_path)
if not entries:
raise BenchmarkError("eval manifest contains no inputs")
out = Path(output_dir)
out.mkdir(parents=True, exist_ok=True)
refs = Path(references_dir) if references_dir else _infer_references_dir(eval_path)
inputs = [entry.image_path for entry in entries]
datasets = sorted({entry.dataset for entry in entries})
slice_counts = _slice_counts(entries)
preflight_report_path: str | None = None
preflight_passed: bool | None = None
if run_preflight:
preflight_dir = out / "preflight"
preflight = run_claim_preflight(
eval_path,
preflight_dir,
baselines=baselines,
references_dir=refs,
candidate_model_config=candidate_model_config,
fallback_model_config=fallback_model_config,
train_manifests=train_manifests or [],
validation_config=validation_config,
gorkhapatra_review_audits=gorkhapatra_review_audits or [],
require_claim_ready_eval_pack=require_claim_ready_eval_pack,
require_trained_recognizer=require_trained_recognizer,
require_model_admission=require_model_admission,
)
preflight_report_path = str(preflight_dir / "preflight.json")
preflight_passed = preflight.passed
if not preflight.passed:
raise BenchmarkError(f"preflight failed; see {preflight_report_path}")
benchmark_dir = out / "benchmark"
results = run_benchmark(
inputs,
benchmark_dir,
baselines=baselines,
references_dir=refs,
eval_manifest=eval_path,
candidate_model_config=candidate_model_config,
fallback_engine=fallback_engine,
fallback_model_config=fallback_model_config,
low_confidence_threshold=low_confidence_threshold,
fallback_min_quality_score=fallback_min_quality_score,
)
status_counts = _status_counts(results)
validation_status = None
validation_report_path: str | None = None
if run_validation:
validation_dir = out / "validation"
validation = validate_claim(
benchmark_dir / "report.json",
validation_dir,
config_path=validation_config,
eval_manifest=eval_path,
train_manifests=train_manifests or [],
candidate_model_config=candidate_model_config,
fallback_model_config=fallback_model_config,
)
validation_status = validation.claim_status
validation_report_path = str(validation_dir / "validation.json")
report = ExperimentReport(
experiment_id=out.name or "experiment",
output_dir=str(out),
eval_manifest=str(eval_path),
references_dir=str(refs) if refs else None,
baselines=baselines,
input_count=len(inputs),
datasets=datasets,
slice_counts=slice_counts,
benchmark_report=str(benchmark_dir / "report.json"),
benchmark_status_counts=status_counts,
preflight_report=preflight_report_path,
preflight_passed=preflight_passed,
validation_report=validation_report_path,
validation_status=validation_status,
environment=_environment(
eval_path,
refs,
validation_config,
train_manifests or [],
candidate_model_config,
fallback_engine,
fallback_model_config,
low_confidence_threshold,
fallback_min_quality_score,
),
)
_write_experiment_report(report, out)
return report
def _infer_references_dir(eval_manifest: Path) -> Path | None:
candidate = eval_manifest.parent.parent / "references"
return candidate if candidate.exists() else None
def _status_counts(results: list[BenchmarkResult]) -> dict[str, int]:
counts: dict[str, int] = {}
for result in results:
counts[result.status] = counts.get(result.status, 0) + 1
return dict(sorted(counts.items()))
def _slice_counts(entries: list[ManifestEntry]) -> dict[str, int]:
counts: dict[str, int] = {}
seen: set[tuple[str, str]] = set()
for entry in entries:
metadata = entry.metadata or {}
values: set[str] = set()
for key in ("slice", "script", "document_type", "degradation", "language"):
value = metadata.get(key)
if isinstance(value, str) and value:
values.add(value)
for key in ("slices", "scripts", "tags"):
value = metadata.get(key)
if isinstance(value, list):
values.update(str(item) for item in value if str(item))
elif isinstance(value, str) and value:
values.add(value)
for slice_name in sorted(values):
pair = (entry.sample_id, slice_name)
if pair in seen:
continue
seen.add(pair)
counts[slice_name] = counts.get(slice_name, 0) + 1
return dict(sorted(counts.items()))
def _environment(
eval_manifest: Path,
references_dir: Path | None,
validation_config: str | Path | None,
train_manifests: list[str | Path],
candidate_model_config: str | Path | None,
fallback_engine: str | None,
fallback_model_config: str | Path | None,
low_confidence_threshold: float,
fallback_min_quality_score: float | None,
) -> dict[str, Any]:
payload: dict[str, Any] = {
"created_at_utc": datetime.now(UTC).isoformat(),
"ocrtech_version": __version__,
"python": platform.python_version(),
"platform": platform.platform(),
"eval_manifest_sha256": sha256_file(eval_manifest) if eval_manifest.exists() else None,
"references_dir": str(references_dir) if references_dir else None,
"validation_config": str(validation_config) if validation_config else None,
"validation_config_sha256": sha256_file(Path(validation_config)) if validation_config and Path(validation_config).exists() else None,
"candidate_model_config": str(candidate_model_config) if candidate_model_config else None,
"candidate_model_config_sha256": sha256_file(Path(candidate_model_config)) if candidate_model_config and Path(candidate_model_config).exists() else None,
"fallback_engine": fallback_engine,
"fallback_model_config": str(fallback_model_config) if fallback_model_config else None,
"fallback_model_config_sha256": sha256_file(Path(fallback_model_config)) if fallback_model_config and Path(fallback_model_config).exists() else None,
"low_confidence_threshold": low_confidence_threshold,
"fallback_min_quality_score": fallback_min_quality_score,
"train_manifests": [],
}
for manifest in train_manifests:
path = Path(manifest)
payload["train_manifests"].append({"path": str(path), "sha256": sha256_file(path) if path.exists() else None})
return payload
def _write_experiment_report(report: ExperimentReport, output_dir: Path) -> None:
(output_dir / "experiment.json").write_text(json.dumps(report.to_dict(), ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
lines = [
"# OCR Experiment",
"",
f"Experiment: `{report.experiment_id}`",
f"Eval manifest: `{report.eval_manifest}`",
f"References: `{report.references_dir or ''}`",
f"Inputs: `{report.input_count}`",
f"Datasets: `{', '.join(report.datasets)}`",
f"Baselines: `{', '.join(report.baselines)}`",
f"Preflight: `{report.preflight_passed if report.preflight_passed is not None else 'not_run'}`",
f"Benchmark report: `{report.benchmark_report}`",
f"Validation status: `{report.validation_status or 'not_run'}`",
"",
"## Benchmark Status",
"",
]
lines.extend(f"- `{key}`: {value}" for key, value in report.benchmark_status_counts.items())
if not report.benchmark_status_counts:
lines.append("- none")
lines.extend(["", "## Slice Coverage", ""])
lines.extend(f"- `{key}`: {value}" for key, value in report.slice_counts.items())
if not report.slice_counts:
lines.append("- none")
(output_dir / "experiment.md").write_text("\n".join(lines) + "\n", encoding="utf-8")