verify.py

"""Score a classifier config over a named pool and write its eval artifact.

    python verify.py                                  # baseline on VAL5000
    python verify.py --classifier classifier_tight_fpr.json
    python verify.py --pool CALIB1000

Writes eval.json for the baseline config and eval_tight_fpr.json for the
tight-FPR one, unless --out says otherwise.
"""
import argparse
import json
import sys
from pathlib import Path

import torch

sys.path.insert(0, str(Path(__file__).resolve().parent))  # repo root, for `common`
from common import BACKBONE, device, f1_at, load_pool, score_pool, write_artifact  # noqa: E402
from common.models import load_backbone  # noqa: E402
from common.pools import VAL5000, by_name  # noqa: E402

HERE = Path(__file__).resolve().parent


def out_path_for(classifier: Path) -> Path:
    """classifier.json -> eval.json; classifier_tight_fpr.json -> eval_tight_fpr.json."""
    suffix = classifier.stem[len('classifier'):]
    return classifier.parent / f'eval{suffix}.json'


def main():
    ap = argparse.ArgumentParser(description=__doc__)
    ap.add_argument('--classifier', type=Path, default=HERE / 'classifier.json')
    ap.add_argument('--backbone', default=BACKBONE)
    ap.add_argument('--pool', default=VAL5000.name)
    ap.add_argument('--out', type=Path, default=None)
    args = ap.parse_args()

    dev = device()
    pool = by_name(args.pool)
    c = json.loads(args.classifier.read_text())
    print(f'[init] loading {args.backbone}', flush=True)
    backbone = load_backbone(args.backbone).to(dev)

    print(f'[pool] {pool.name}', flush=True)
    loaded = load_pool(pool, dev)
    pos = torch.tensor(c['pos_dims'], dtype=torch.long, device=dev)
    neg = torch.tensor(c['neg_dims'], dtype=torch.long, device=dev)

    scores, _ = score_pool(backbone, loaded, pos, neg)
    m = f1_at(scores, loaded.labels, c['threshold'])
    print(f'[verify] F1={m.f1:.4f}  P={m.precision:.4f}  R={m.recall:.4f}', flush=True)

    path = args.out or out_path_for(args.classifier)
    write_artifact(
        path, {'metrics': {k: round(v, 4) if k != 'threshold' else v
                           for k, v in m.asdict().items()}},
        generator='verify.py', classifier=args.classifier,
        pool_info=loaded.provenance(),
        task='image-level person presence (binary)',
        protocol='live backbone forward at 768 px, no feature caching')
    print(f'[done] wrote {path}', flush=True)


if __name__ == '__main__':
    main()