File size: 4,465 Bytes
1ef1de7
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
from __future__ import annotations

import json
from pathlib import Path

import pytest

from ocrtech.evalpack import create_eval_pack
from ocrtech.experiments import run_experiment
from ocrtech.errors import BenchmarkError
from ocrtech.models import write_model_card


def test_run_experiment_writes_benchmark_and_validation_reports(tmp_path: Path) -> None:
    pack = create_eval_pack(tmp_path / "pack", input_format="text", count_per_template=1, degradations=["clean"])
    config = tmp_path / "validation-config.json"
    config.write_text(
        json.dumps(
            {
                "candidate": "ours",
                "min_samples": 6,
                "required_slices": {"nepali": 6, "english": 6, "multi_page": 1, "complex_table": 1},
                "gates": [{"baseline": "ours", "metric": "quality_score", "direction": "higher", "min_absolute_margin": 0.0, "require_ci_above_zero": False}],
            }
        ),
        encoding="utf-8",
    )

    report = run_experiment(
        pack.manifest_path,
        tmp_path / "experiment",
        baselines=["ours"],
        references_dir=tmp_path / "pack" / "references",
        validation_config=config,
    )

    assert report.input_count == 6
    assert report.benchmark_status_counts["ok"] == 6
    assert report.validation_status == "validated"
    assert (tmp_path / "experiment" / "benchmark" / "report.json").exists()
    assert (tmp_path / "experiment" / "validation" / "validation.json").exists()
    assert (tmp_path / "experiment" / "experiment.json").exists()


def test_run_experiment_can_run_preflight_before_benchmark(tmp_path: Path) -> None:
    pack = create_eval_pack(tmp_path / "pack", input_format="text", count_per_template=1, degradations=["clean"])
    card = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture")
    config = tmp_path / "validation-config.json"
    config.write_text(
        json.dumps(
            {
                "candidate": "ours",
                "min_samples": 6,
                "required_slices": {"nepali": 6, "english": 6, "multi_page": 1, "complex_table": 1},
                "gates": [
                    {
                        "baseline": "ours",
                        "metric": "quality_score",
                        "direction": "higher",
                        "min_absolute_margin": 0.0,
                        "require_ci_above_zero": False,
                    }
                ],
            }
        ),
        encoding="utf-8",
    )

    report = run_experiment(
        pack.manifest_path,
        tmp_path / "experiment",
        baselines=["ours"],
        references_dir=tmp_path / "pack" / "references",
        validation_config=config,
        candidate_model_config=card,
        run_preflight=True,
        require_claim_ready_eval_pack=False,
    )

    payload = json.loads((tmp_path / "experiment" / "experiment.json").read_text(encoding="utf-8"))
    assert report.preflight_passed is True
    assert report.preflight_report == str(tmp_path / "experiment" / "preflight" / "preflight.json")
    assert payload["preflight_passed"] is True
    assert Path(payload["preflight_report"]).exists()


def test_run_experiment_preflight_stops_impossible_claim_run_before_benchmark(tmp_path: Path) -> None:
    pack = create_eval_pack(tmp_path / "pack", input_format="text", count_per_template=1, degradations=["clean"])
    card = write_model_card(tmp_path / "model.json", model_id="ocrtech-sidecar", backend="sidecar", base_model="fixture")
    config = tmp_path / "validation-config.json"
    config.write_text(
        json.dumps(
            {
                "candidate": "ours",
                "min_samples": 1,
                "gates": [{"baseline": "tesseract", "metric": "cer", "direction": "lower"}],
            }
        ),
        encoding="utf-8",
    )

    with pytest.raises(BenchmarkError, match="preflight failed"):
        run_experiment(
            pack.manifest_path,
            tmp_path / "experiment",
            baselines=["ours"],
            references_dir=tmp_path / "pack" / "references",
            validation_config=config,
            candidate_model_config=card,
            run_preflight=True,
            require_claim_ready_eval_pack=False,
        )

    assert (tmp_path / "experiment" / "preflight" / "preflight.json").exists()
    assert not (tmp_path / "experiment" / "benchmark" / "report.json").exists()