app/training/validate.py

"""Check a labelling run against the gate before anyone trains on it.

    .venv/bin/python -m app.training.validate cattle_weight data/weight.jsonl
    .venv/bin/python -m app.training.validate cattle_bcs data/bcs.jsonl

What this can tell you is whether the **data** could support a model that
passes: enough ground truth, enough farms, enough breeds, and coverage across
the range rather than a pile in the middle. What it cannot tell you is whether a
model trained on it hits the metric — that needs the model. Exit code 0 means
"worth training on", never "ready to promote".

Run it while collection is still happening. A dataset that turns out to be four
hundred animals from one farm is recoverable in month two and not in month six.
"""

from __future__ import annotations

import argparse
import json
import sys
from collections import Counter
from pathlib import Path

from pydantic import ValidationError

from app.training.gates import PROMOTION_GATES, PromotionGate
from app.training.schema import BcsSample, WeightSample

_MODELS = {"cattle_weight": WeightSample, "cattle_bcs": BcsSample}


def _bucket(sample) -> str:
    if isinstance(sample, WeightSample):
        return sample.weight_band
    return f"bcs_{sample.consensus:.1f}"


def _load(path: Path, model) -> tuple[list, list[str]]:
    samples, problems = [], []
    for number, line in enumerate(path.read_text().splitlines(), start=1):
        line = line.strip()
        if not line or line.startswith("#"):
            continue
        try:
            samples.append(model.model_validate(json.loads(line)))
        except (json.JSONDecodeError, ValidationError) as exc:
            first = str(exc).splitlines()[-1].strip()
            problems.append(f"line {number}: {first}")
    return samples, problems


def report(samples: list, gate: PromotionGate) -> list[tuple[bool, str]]:
    truth = [s for s in samples if s.is_ground_truth]
    farms = {s.farm_id for s in truth}
    animals = {s.animal_id for s in truth}
    breeds = {s.breed for s in truth}
    buckets = Counter(_bucket(s) for s in truth)
    total = len(truth) or 1

    thin = sorted(
        b for b, n in buckets.items() if n / total < gate.min_share_per_bucket
    )

    lines = [
        (len(truth) >= gate.min_ground_truth_samples,
         f"ground-truth samples: {len(truth)} of {gate.min_ground_truth_samples} "
         f"({len(samples) - len(truth)} excluded as estimates or unusable)"),
        (len(animals) >= gate.min_animals,
         f"distinct animals: {len(animals)} of {gate.min_animals}"),
        (len(farms) >= gate.min_farms,
         f"farms: {len(farms)} of {gate.min_farms}"),
        (len(breeds) >= gate.min_breeds,
         f"breeds: {len(breeds)} of {gate.min_breeds} — {', '.join(sorted(breeds)) or 'none'}"),
        (not thin,
         f"buckets under {gate.min_share_per_bucket:.0%}: "
         f"{', '.join(thin) if thin else 'none'}"),
        (len(farms) >= gate.min_farms + 2,
         f"farms to spare for the held-out split: {max(len(farms) - gate.min_farms, 0)} of 2"),
    ]
    return lines


def main(argv: list[str] | None = None) -> int:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("capability", choices=sorted(_MODELS))
    parser.add_argument("manifest", type=Path)
    args = parser.parse_args(argv)

    gate = PROMOTION_GATES[args.capability]
    samples, problems = _load(args.manifest, _MODELS[args.capability])

    print(f"{args.manifest} — {args.capability}")
    if problems:
        print(f"\n  {len(problems)} record(s) rejected:")
        for problem in problems[:20]:
            print(f"    ✗ {problem}")
        if len(problems) > 20:
            print(f"    … and {len(problems) - 20} more")

    print()
    lines = report(samples, gate)
    for passed, text in lines:
        print(f"  {'✓' if passed else '✗'} {text}")

    print(f"\n  held-out split: {gate.holdout}")
    print(f"  metric to beat: {gate.metric}")
    print(f"                  {gate.threshold}")

    ready = all(passed for passed, _ in lines) and not problems
    verdict = (
        "The data could support a model that passes. Train, then measure."
        if ready
        else "Not enough data yet. Keep collecting."
    )
    print(f"\n  {verdict}")
    return 0 if ready else 1


if __name__ == "__main__":
    sys.exit(main())