File size: 2,486 Bytes
f5498f9 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 | """Score a classifier config over a named pool and write its eval artifact.
python verify.py # baseline on VAL5000
python verify.py --classifier classifier_tight_fpr.json
python verify.py --pool CALIB1000
Writes eval.json for the baseline config and eval_tight_fpr.json for the
tight-FPR one, unless --out says otherwise.
"""
import argparse
import json
import sys
from pathlib import Path
import torch
sys.path.insert(0, str(Path(__file__).resolve().parent)) # repo root, for `common`
from common import BACKBONE, device, f1_at, load_pool, score_pool, write_artifact # noqa: E402
from common.models import load_backbone # noqa: E402
from common.pools import VAL5000, by_name # noqa: E402
HERE = Path(__file__).resolve().parent
def out_path_for(classifier: Path) -> Path:
"""classifier.json -> eval.json; classifier_tight_fpr.json -> eval_tight_fpr.json."""
suffix = classifier.stem[len('classifier'):]
return classifier.parent / f'eval{suffix}.json'
def main():
ap = argparse.ArgumentParser(description=__doc__)
ap.add_argument('--classifier', type=Path, default=HERE / 'classifier.json')
ap.add_argument('--backbone', default=BACKBONE)
ap.add_argument('--pool', default=VAL5000.name)
ap.add_argument('--out', type=Path, default=None)
args = ap.parse_args()
dev = device()
pool = by_name(args.pool)
c = json.loads(args.classifier.read_text())
print(f'[init] loading {args.backbone}', flush=True)
backbone = load_backbone(args.backbone).to(dev)
print(f'[pool] {pool.name}', flush=True)
loaded = load_pool(pool, dev)
pos = torch.tensor(c['pos_dims'], dtype=torch.long, device=dev)
neg = torch.tensor(c['neg_dims'], dtype=torch.long, device=dev)
scores, _ = score_pool(backbone, loaded, pos, neg)
m = f1_at(scores, loaded.labels, c['threshold'])
print(f'[verify] F1={m.f1:.4f} P={m.precision:.4f} R={m.recall:.4f}', flush=True)
path = args.out or out_path_for(args.classifier)
write_artifact(
path, {'metrics': {k: round(v, 4) if k != 'threshold' else v
for k, v in m.asdict().items()}},
generator='verify.py', classifier=args.classifier,
pool_info=loaded.provenance(),
task='image-level person presence (binary)',
protocol='live backbone forward at 768 px, no feature caching')
print(f'[done] wrote {path}', flush=True)
if __name__ == '__main__':
main()
|