Add 195 files (runs/hf-recognizer-mixed-v2-full/finalized/package/artifacts/model/model.safetensors ...)
1ef1de7 verified Download tests/test_experiments.py from prosperious/ocr-tech-k2: direct link, hf CLI and curl.
- Browser
- Download file 4.47 kB
-
https://huggingface.co/prosperious/ocr-tech-k2/resolve/main/tests/test_experiments.py
- Command line
-
hf download hf://prosperious/ocr-tech-k2/tests/test_experiments.py
-
curl -L -o test_experiments.py https://huggingface.co/prosperious/ocr-tech-k2/resolve/main/tests/test_experiments.py
4.47 kB
| from __future__ import annotations | |
| import json | |
| from pathlib import Path | |
| import pytest | |
| from ocrtech.evalpack import create_eval_pack | |
| from ocrtech.experiments import run_experiment | |
| from ocrtech.errors import BenchmarkError | |
| from ocrtech.models import write_model_card | |
| def test_run_experiment_writes_benchmark_and_validation_reports(tmp_path: Path) -> None: | |
| pack = create_eval_pack(tmp_path / "pack", input_format="text", count_per_template=1, degradations=["clean"]) | |
| config = tmp_path / "validation-config.json" | |
| config.write_text( | |
| json.dumps( | |
| { | |
| "candidate": "ours", | |
| "min_samples": 6, | |
| "required_slices": {"nepali": 6, "english": 6, "multi_page": 1, "complex_table": 1}, | |
| "gates": [{"baseline": "ours", "metric": "quality_score", "direction": "higher", "min_absolute_margin": 0.0, "require_ci_above_zero": False}], | |
| } | |
| ), | |
| encoding="utf-8", | |
| ) | |
| report = run_experiment( | |
| pack.manifest_path, | |
| tmp_path / "experiment", | |
| baselines=["ours"], | |
| references_dir=tmp_path / "pack" / "references", | |
| validation_config=config, | |
| ) | |
| assert report.input_count == 6 | |
| assert report.benchmark_status_counts["ok"] == 6 | |
| assert report.validation_status == "validated" | |
| assert (tmp_path / "experiment" / "benchmark" / "report.json").exists() | |
| assert (tmp_path / "experiment" / "validation" / "validation.json").exists() | |
| assert (tmp_path / "experiment" / "experiment.json").exists() | |
| def test_run_experiment_can_run_preflight_before_benchmark(tmp_path: Path) -> None: | |
| pack = create_eval_pack(tmp_path / "pack", input_format="text", count_per_template=1, degradations=["clean"]) | |
| card = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture") | |
| config = tmp_path / "validation-config.json" | |
| config.write_text( | |
| json.dumps( | |
| { | |
| "candidate": "ours", | |
| "min_samples": 6, | |
| "required_slices": {"nepali": 6, "english": 6, "multi_page": 1, "complex_table": 1}, | |
| "gates": [ | |
| { | |
| "baseline": "ours", | |
| "metric": "quality_score", | |
| "direction": "higher", | |
| "min_absolute_margin": 0.0, | |
| "require_ci_above_zero": False, | |
| } | |
| ], | |
| } | |
| ), | |
| encoding="utf-8", | |
| ) | |
| report = run_experiment( | |
| pack.manifest_path, | |
| tmp_path / "experiment", | |
| baselines=["ours"], | |
| references_dir=tmp_path / "pack" / "references", | |
| validation_config=config, | |
| candidate_model_config=card, | |
| run_preflight=True, | |
| require_claim_ready_eval_pack=False, | |
| ) | |
| payload = json.loads((tmp_path / "experiment" / "experiment.json").read_text(encoding="utf-8")) | |
| assert report.preflight_passed is True | |
| assert report.preflight_report == str(tmp_path / "experiment" / "preflight" / "preflight.json") | |
| assert payload["preflight_passed"] is True | |
| assert Path(payload["preflight_report"]).exists() | |
| def test_run_experiment_preflight_stops_impossible_claim_run_before_benchmark(tmp_path: Path) -> None: | |
| pack = create_eval_pack(tmp_path / "pack", input_format="text", count_per_template=1, degradations=["clean"]) | |
| card = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture") | |
| config = tmp_path / "validation-config.json" | |
| config.write_text( | |
| json.dumps( | |
| { | |
| "candidate": "ours", | |
| "min_samples": 1, | |
| "gates": [{"baseline": "tesseract", "metric": "cer", "direction": "lower"}], | |
| } | |
| ), | |
| encoding="utf-8", | |
| ) | |
| with pytest.raises(BenchmarkError, match="preflight failed"): | |
| run_experiment( | |
| pack.manifest_path, | |
| tmp_path / "experiment", | |
| baselines=["ours"], | |
| references_dir=tmp_path / "pack" / "references", | |
| validation_config=config, | |
| candidate_model_config=card, | |
| run_preflight=True, | |
| require_claim_ready_eval_pack=False, | |
| ) | |
| assert (tmp_path / "experiment" / "preflight" / "preflight.json").exists() | |
| assert not (tmp_path / "experiment" / "benchmark" / "report.json").exists() | |