File size: 1,553 Bytes
d82db76
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
"""Checkpoint selection with no-regression guards (dependency-free so it is unit-testable).

A fine-tune that adds a new domain (handwriting) must not buy its gain with the old domains
(print, English). Guard dev sets keep their step-0 codepoint CER as a baseline; a checkpoint
whose guard CER rises more than ``tol`` above it is ineligible, and eligible checkpoints are
ranked by the mean CER of the remaining (target) dev sets only.
"""

from __future__ import annotations


def split_dev_sets(names: list[str], guards: list[str]) -> list[str]:
    """Target set names (every dev set that is not a guard); all sets when every set is a guard."""
    unknown = sorted(set(guards) - set(names))
    if unknown:
        raise ValueError(f"guard dev sets not among dev sets: {unknown}")
    return [name for name in names if name not in guards] or list(names)


def guard_regressions(cer: dict[str, float], baseline: dict[str, float], tol: float) -> dict[str, float]:
    """Guard sets whose CER rose more than ``tol`` above baseline, mapped to the rise."""
    missing = sorted(set(baseline) - set(cer))
    if missing:
        raise KeyError(f"no CER reported for guard sets: {missing}")
    return {name: round(cer[name] - base, 5) for name, base in baseline.items() if cer[name] > base + tol}


def target_score(cer: dict[str, float], targets: list[str]) -> float:
    """Mean CER over the target sets (lower is better)."""
    if not targets:
        raise ValueError("no target dev sets")
    return sum(cer[name] for name in targets) / len(targets)