"""Checkpoint selection with no-regression guards (dependency-free so it is unit-testable). A fine-tune that adds a new domain (handwriting) must not buy its gain with the old domains (print, English). Guard dev sets keep their step-0 codepoint CER as a baseline; a checkpoint whose guard CER rises more than ``tol`` above it is ineligible, and eligible checkpoints are ranked by the mean CER of the remaining (target) dev sets only. """ from __future__ import annotations def split_dev_sets(names: list[str], guards: list[str]) -> list[str]: """Target set names (every dev set that is not a guard); all sets when every set is a guard.""" unknown = sorted(set(guards) - set(names)) if unknown: raise ValueError(f"guard dev sets not among dev sets: {unknown}") return [name for name in names if name not in guards] or list(names) def guard_regressions(cer: dict[str, float], baseline: dict[str, float], tol: float) -> dict[str, float]: """Guard sets whose CER rose more than ``tol`` above baseline, mapped to the rise.""" missing = sorted(set(baseline) - set(cer)) if missing: raise KeyError(f"no CER reported for guard sets: {missing}") return {name: round(cer[name] - base, 5) for name, base in baseline.items() if cer[name] > base + tol} def target_score(cer: dict[str, float], targets: list[str]) -> float: """Mean CER over the target sets (lower is better).""" if not targets: raise ValueError("no target dev sets") return sum(cer[name] for name in targets) / len(targets)