prosperious's picture
Add 1000 files (ocr-tech/tests/test_audit_limbu_book_missed_line_review_coverage.py ...)
d82db76 verified
Raw History Blame Contribute Delete
1.55 kB
"""Checkpoint selection with no-regression guards (dependency-free so it is unit-testable).
A fine-tune that adds a new domain (handwriting) must not buy its gain with the old domains
(print, English). Guard dev sets keep their step-0 codepoint CER as a baseline; a checkpoint
whose guard CER rises more than ``tol`` above it is ineligible, and eligible checkpoints are
ranked by the mean CER of the remaining (target) dev sets only.
"""
from __future__ import annotations
def split_dev_sets(names: list[str], guards: list[str]) -> list[str]:
"""Target set names (every dev set that is not a guard); all sets when every set is a guard."""
unknown = sorted(set(guards) - set(names))
if unknown:
raise ValueError(f"guard dev sets not among dev sets: {unknown}")
return [name for name in names if name not in guards] or list(names)
def guard_regressions(cer: dict[str, float], baseline: dict[str, float], tol: float) -> dict[str, float]:
"""Guard sets whose CER rose more than ``tol`` above baseline, mapped to the rise."""
missing = sorted(set(baseline) - set(cer))
if missing:
raise KeyError(f"no CER reported for guard sets: {missing}")
return {name: round(cer[name] - base, 5) for name, base in baseline.items() if cer[name] > base + tol}
def target_score(cer: dict[str, float], targets: list[str]) -> float:
"""Mean CER over the target sets (lower is better)."""
if not targets:
raise ValueError("no target dev sets")
return sum(cer[name] for name in targets) / len(targets)