File size: 1,803 Bytes
1c0c94d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 | """CLI: python -m eval.run [dataset.json] (default: eval/sample_dataset.json)
Runs the real pipeline (downloads YOLO weights on first use) over the labelled set and
prints the metrics table used in the pitch.
"""
from __future__ import annotations
import sys
from eval.harness import EvalReport, evaluate, load_dataset
from eval.metrics import PRF
def _table(metrics: dict[str, PRF]) -> str:
header = f"{'type':<28}{'P':>6}{'R':>6}{'F1':>6}{'TP':>5}{'FP':>5}{'FN':>5}"
rows = [
f"{k:<28}{m.precision:6.2f}{m.recall:6.2f}{m.f1:6.2f}{m.tp:5d}{m.fp:5d}{m.fn:5d}"
for k, m in metrics.items()
]
return "\n".join([header, *rows])
def report(rep: EvalReport) -> str:
out = [
f"Samples: {rep.n} mean latency: {rep.mean_latency_s * 1000:.0f} ms/image",
"",
"== rule + VLM (routed; auto/VLM-confirmed only) ==",
_table(rep.routed),
f"macro-F1: {rep.macro_f1_routed:.3f}",
"",
"== rule-only (ablation; no VLM filtering) ==",
_table(rep.rule_only),
f"macro-F1: {rep.macro_f1_rule_only:.3f}",
"",
f"Dispositions: {rep.dispositions}",
]
if rep.plate_whole_accuracy is not None:
out.append(
f"Plate whole-accuracy: {rep.plate_whole_accuracy:.2f} "
f"char-accuracy: {rep.plate_char_accuracy:.2f}"
)
out.append(
"\nNote: object-detection mAP needs bbox-level labels; this harness reports "
"violation-level metrics. Use the base YOLO model's COCO mAP for detector mAP."
)
return "\n".join(out)
def main(path: str = "eval/sample_dataset.json") -> None:
print(report(evaluate(load_dataset(path))))
if __name__ == "__main__":
main(sys.argv[1] if len(sys.argv) > 1 else "eval/sample_dataset.json")
|