ocr-tech-k2 / tests /test_experiments.py
prosperious's picture
Add 195 files (runs/hf-recognizer-mixed-v2-full/finalized/package/artifacts/model/model.safetensors ...)
1ef1de7 verified
Raw History Blame Contribute Delete
4.47 kB
from __future__ import annotations
import json
from pathlib import Path
import pytest
from ocrtech.evalpack import create_eval_pack
from ocrtech.experiments import run_experiment
from ocrtech.errors import BenchmarkError
from ocrtech.models import write_model_card
def test_run_experiment_writes_benchmark_and_validation_reports(tmp_path: Path) -> None:
pack = create_eval_pack(tmp_path / "pack", input_format="text", count_per_template=1, degradations=["clean"])
config = tmp_path / "validation-config.json"
config.write_text(
json.dumps(
{
"candidate": "ours",
"min_samples": 6,
"required_slices": {"nepali": 6, "english": 6, "multi_page": 1, "complex_table": 1},
"gates": [{"baseline": "ours", "metric": "quality_score", "direction": "higher", "min_absolute_margin": 0.0, "require_ci_above_zero": False}],
}
),
encoding="utf-8",
)
report = run_experiment(
pack.manifest_path,
tmp_path / "experiment",
baselines=["ours"],
references_dir=tmp_path / "pack" / "references",
validation_config=config,
)
assert report.input_count == 6
assert report.benchmark_status_counts["ok"] == 6
assert report.validation_status == "validated"
assert (tmp_path / "experiment" / "benchmark" / "report.json").exists()
assert (tmp_path / "experiment" / "validation" / "validation.json").exists()
assert (tmp_path / "experiment" / "experiment.json").exists()
def test_run_experiment_can_run_preflight_before_benchmark(tmp_path: Path) -> None:
pack = create_eval_pack(tmp_path / "pack", input_format="text", count_per_template=1, degradations=["clean"])
card = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture")
config = tmp_path / "validation-config.json"
config.write_text(
json.dumps(
{
"candidate": "ours",
"min_samples": 6,
"required_slices": {"nepali": 6, "english": 6, "multi_page": 1, "complex_table": 1},
"gates": [
{
"baseline": "ours",
"metric": "quality_score",
"direction": "higher",
"min_absolute_margin": 0.0,
"require_ci_above_zero": False,
}
],
}
),
encoding="utf-8",
)
report = run_experiment(
pack.manifest_path,
tmp_path / "experiment",
baselines=["ours"],
references_dir=tmp_path / "pack" / "references",
validation_config=config,
candidate_model_config=card,
run_preflight=True,
require_claim_ready_eval_pack=False,
)
payload = json.loads((tmp_path / "experiment" / "experiment.json").read_text(encoding="utf-8"))
assert report.preflight_passed is True
assert report.preflight_report == str(tmp_path / "experiment" / "preflight" / "preflight.json")
assert payload["preflight_passed"] is True
assert Path(payload["preflight_report"]).exists()
def test_run_experiment_preflight_stops_impossible_claim_run_before_benchmark(tmp_path: Path) -> None:
pack = create_eval_pack(tmp_path / "pack", input_format="text", count_per_template=1, degradations=["clean"])
card = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture")
config = tmp_path / "validation-config.json"
config.write_text(
json.dumps(
{
"candidate": "ours",
"min_samples": 1,
"gates": [{"baseline": "tesseract", "metric": "cer", "direction": "lower"}],
}
),
encoding="utf-8",
)
with pytest.raises(BenchmarkError, match="preflight failed"):
run_experiment(
pack.manifest_path,
tmp_path / "experiment",
baselines=["ours"],
references_dir=tmp_path / "pack" / "references",
validation_config=config,
candidate_model_config=card,
run_preflight=True,
require_claim_ready_eval_pack=False,
)
assert (tmp_path / "experiment" / "preflight" / "preflight.json").exists()
assert not (tmp_path / "experiment" / "benchmark" / "report.json").exists()