from __future__ import annotations import json from pathlib import Path import pytest from ocrtech.evalpack import create_eval_pack from ocrtech.experiments import run_experiment from ocrtech.errors import BenchmarkError from ocrtech.models import write_model_card def test_run_experiment_writes_benchmark_and_validation_reports(tmp_path: Path) -> None: pack = create_eval_pack(tmp_path / "pack", input_format="text", count_per_template=1, degradations=["clean"]) config = tmp_path / "validation-config.json" config.write_text( json.dumps( { "candidate": "ours", "min_samples": 6, "required_slices": {"nepali": 6, "english": 6, "multi_page": 1, "complex_table": 1}, "gates": [{"baseline": "ours", "metric": "quality_score", "direction": "higher", "min_absolute_margin": 0.0, "require_ci_above_zero": False}], } ), encoding="utf-8", ) report = run_experiment( pack.manifest_path, tmp_path / "experiment", baselines=["ours"], references_dir=tmp_path / "pack" / "references", validation_config=config, ) assert report.input_count == 6 assert report.benchmark_status_counts["ok"] == 6 assert report.validation_status == "validated" assert (tmp_path / "experiment" / "benchmark" / "report.json").exists() assert (tmp_path / "experiment" / "validation" / "validation.json").exists() assert (tmp_path / "experiment" / "experiment.json").exists() def test_run_experiment_can_run_preflight_before_benchmark(tmp_path: Path) -> None: pack = create_eval_pack(tmp_path / "pack", input_format="text", count_per_template=1, degradations=["clean"]) card = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture") config = tmp_path / "validation-config.json" config.write_text( json.dumps( { "candidate": "ours", "min_samples": 6, "required_slices": {"nepali": 6, "english": 6, "multi_page": 1, "complex_table": 1}, "gates": [ { "baseline": "ours", "metric": "quality_score", "direction": "higher", "min_absolute_margin": 0.0, "require_ci_above_zero": False, } ], } ), encoding="utf-8", ) report = run_experiment( pack.manifest_path, tmp_path / "experiment", baselines=["ours"], references_dir=tmp_path / "pack" / "references", validation_config=config, candidate_model_config=card, run_preflight=True, require_claim_ready_eval_pack=False, ) payload = json.loads((tmp_path / "experiment" / "experiment.json").read_text(encoding="utf-8")) assert report.preflight_passed is True assert report.preflight_report == str(tmp_path / "experiment" / "preflight" / "preflight.json") assert payload["preflight_passed"] is True assert Path(payload["preflight_report"]).exists() def test_run_experiment_preflight_stops_impossible_claim_run_before_benchmark(tmp_path: Path) -> None: pack = create_eval_pack(tmp_path / "pack", input_format="text", count_per_template=1, degradations=["clean"]) card = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture") config = tmp_path / "validation-config.json" config.write_text( json.dumps( { "candidate": "ours", "min_samples": 1, "gates": [{"baseline": "tesseract", "metric": "cer", "direction": "lower"}], } ), encoding="utf-8", ) with pytest.raises(BenchmarkError, match="preflight failed"): run_experiment( pack.manifest_path, tmp_path / "experiment", baselines=["ours"], references_dir=tmp_path / "pack" / "references", validation_config=config, candidate_model_config=card, run_preflight=True, require_claim_ready_eval_pack=False, ) assert (tmp_path / "experiment" / "preflight" / "preflight.json").exists() assert not (tmp_path / "experiment" / "benchmark" / "report.json").exists()