Remove bbox detection route artifacts
Browse files
scripts/evaluate_yolo_detection.py
DELETED
|
@@ -1,198 +0,0 @@
|
|
| 1 |
-
"""Evaluate a YOLO detector on a dataset split with fixed-threshold metrics."""
|
| 2 |
-
|
| 3 |
-
from __future__ import annotations
|
| 4 |
-
|
| 5 |
-
import argparse
|
| 6 |
-
import json
|
| 7 |
-
from pathlib import Path
|
| 8 |
-
|
| 9 |
-
import yaml
|
| 10 |
-
from PIL import Image
|
| 11 |
-
|
| 12 |
-
|
| 13 |
-
def parse_args() -> argparse.Namespace:
|
| 14 |
-
parser = argparse.ArgumentParser(description=__doc__)
|
| 15 |
-
parser.add_argument("--weights", required=True)
|
| 16 |
-
parser.add_argument("--data", required=True)
|
| 17 |
-
parser.add_argument("--split", default="test", choices=["train", "val", "test"])
|
| 18 |
-
parser.add_argument("--imgsz", type=int, default=640)
|
| 19 |
-
parser.add_argument("--conf", type=float, default=0.25)
|
| 20 |
-
parser.add_argument("--iou", type=float, default=0.5)
|
| 21 |
-
parser.add_argument("--device", default="0")
|
| 22 |
-
parser.add_argument("--output", required=True)
|
| 23 |
-
return parser.parse_args()
|
| 24 |
-
|
| 25 |
-
|
| 26 |
-
def xywhn_to_xyxy(row: list[float], width: int, height: int) -> list[float]:
|
| 27 |
-
_, x, y, w, h = row[:5]
|
| 28 |
-
cx = x * width
|
| 29 |
-
cy = y * height
|
| 30 |
-
bw = w * width
|
| 31 |
-
bh = h * height
|
| 32 |
-
return [cx - bw / 2, cy - bh / 2, cx + bw / 2, cy + bh / 2]
|
| 33 |
-
|
| 34 |
-
|
| 35 |
-
def box_iou(a: list[float], b: list[float]) -> float:
|
| 36 |
-
x1 = max(a[0], b[0])
|
| 37 |
-
y1 = max(a[1], b[1])
|
| 38 |
-
x2 = min(a[2], b[2])
|
| 39 |
-
y2 = min(a[3], b[3])
|
| 40 |
-
inter = max(0.0, x2 - x1) * max(0.0, y2 - y1)
|
| 41 |
-
area_a = max(0.0, a[2] - a[0]) * max(0.0, a[3] - a[1])
|
| 42 |
-
area_b = max(0.0, b[2] - b[0]) * max(0.0, b[3] - b[1])
|
| 43 |
-
denom = area_a + area_b - inter
|
| 44 |
-
return inter / denom if denom else 0.0
|
| 45 |
-
|
| 46 |
-
|
| 47 |
-
def read_label(label_path: Path, width: int, height: int) -> list[list[float]]:
|
| 48 |
-
if not label_path.exists():
|
| 49 |
-
return []
|
| 50 |
-
boxes: list[list[float]] = []
|
| 51 |
-
for line in label_path.read_text(encoding="utf-8").splitlines():
|
| 52 |
-
if not line.strip():
|
| 53 |
-
continue
|
| 54 |
-
values = [float(part) for part in line.split()]
|
| 55 |
-
boxes.append(xywhn_to_xyxy(values, width, height))
|
| 56 |
-
return boxes
|
| 57 |
-
|
| 58 |
-
|
| 59 |
-
def resolve_split_images(data_yaml: Path, split: str) -> list[Path]:
|
| 60 |
-
data = yaml.safe_load(data_yaml.read_text(encoding="utf-8"))
|
| 61 |
-
root = Path(data.get("path", data_yaml.parent))
|
| 62 |
-
split_value = Path(data[split])
|
| 63 |
-
image_dir = split_value if split_value.is_absolute() else root / split_value
|
| 64 |
-
images = []
|
| 65 |
-
for suffix in ("*.jpg", "*.jpeg", "*.png", "*.tif", "*.tiff"):
|
| 66 |
-
images.extend(image_dir.glob(suffix))
|
| 67 |
-
return sorted(images)
|
| 68 |
-
|
| 69 |
-
|
| 70 |
-
def label_for_image(image_path: Path) -> Path:
|
| 71 |
-
parts = list(image_path.parts)
|
| 72 |
-
for idx, part in enumerate(parts):
|
| 73 |
-
if part == "images":
|
| 74 |
-
parts[idx] = "labels"
|
| 75 |
-
return Path(*parts).with_suffix(".txt")
|
| 76 |
-
return image_path.with_suffix(".txt")
|
| 77 |
-
|
| 78 |
-
|
| 79 |
-
def main() -> None:
|
| 80 |
-
args = parse_args()
|
| 81 |
-
from ultralytics import YOLO
|
| 82 |
-
|
| 83 |
-
image_paths = resolve_split_images(Path(args.data), args.split)
|
| 84 |
-
model = YOLO(args.weights)
|
| 85 |
-
|
| 86 |
-
image_tp = image_fn = image_fp = image_tn = 0
|
| 87 |
-
matched = false_positive_instances = false_negative_instances = 0
|
| 88 |
-
gt_instances = pred_instances = 0
|
| 89 |
-
matched_ious: list[float] = []
|
| 90 |
-
error_images: list[dict[str, object]] = []
|
| 91 |
-
|
| 92 |
-
for image_path in image_paths:
|
| 93 |
-
with Image.open(image_path) as image:
|
| 94 |
-
width, height = image.size
|
| 95 |
-
gt_boxes = read_label(label_for_image(image_path), width, height)
|
| 96 |
-
result = model.predict(
|
| 97 |
-
source=str(image_path),
|
| 98 |
-
imgsz=args.imgsz,
|
| 99 |
-
conf=args.conf,
|
| 100 |
-
device=args.device,
|
| 101 |
-
verbose=False,
|
| 102 |
-
)[0]
|
| 103 |
-
pred_boxes = result.boxes.xyxy.cpu().tolist() if result.boxes is not None else []
|
| 104 |
-
pred_scores = result.boxes.conf.cpu().tolist() if result.boxes is not None else []
|
| 105 |
-
pred_order = sorted(range(len(pred_boxes)), key=lambda i: pred_scores[i], reverse=True)
|
| 106 |
-
|
| 107 |
-
gt_instances += len(gt_boxes)
|
| 108 |
-
pred_instances += len(pred_boxes)
|
| 109 |
-
used_gt: set[int] = set()
|
| 110 |
-
image_matched = 0
|
| 111 |
-
image_false_positives = 0
|
| 112 |
-
|
| 113 |
-
for pred_idx in pred_order:
|
| 114 |
-
best_gt = -1
|
| 115 |
-
best_iou = 0.0
|
| 116 |
-
for gt_idx, gt_box in enumerate(gt_boxes):
|
| 117 |
-
if gt_idx in used_gt:
|
| 118 |
-
continue
|
| 119 |
-
iou = box_iou(pred_boxes[pred_idx], gt_box)
|
| 120 |
-
if iou > best_iou:
|
| 121 |
-
best_iou = iou
|
| 122 |
-
best_gt = gt_idx
|
| 123 |
-
if best_gt >= 0 and best_iou >= args.iou:
|
| 124 |
-
used_gt.add(best_gt)
|
| 125 |
-
matched += 1
|
| 126 |
-
image_matched += 1
|
| 127 |
-
matched_ious.append(best_iou)
|
| 128 |
-
else:
|
| 129 |
-
false_positive_instances += 1
|
| 130 |
-
image_false_positives += 1
|
| 131 |
-
|
| 132 |
-
missed = len(gt_boxes) - image_matched
|
| 133 |
-
false_negative_instances += missed
|
| 134 |
-
if missed or image_false_positives:
|
| 135 |
-
error_images.append(
|
| 136 |
-
{
|
| 137 |
-
"image": str(image_path),
|
| 138 |
-
"gt_instances": len(gt_boxes),
|
| 139 |
-
"pred_instances": len(pred_boxes),
|
| 140 |
-
"matched_instances": image_matched,
|
| 141 |
-
"false_positives": image_false_positives,
|
| 142 |
-
"false_negatives": missed,
|
| 143 |
-
}
|
| 144 |
-
)
|
| 145 |
-
|
| 146 |
-
if gt_boxes:
|
| 147 |
-
if image_matched:
|
| 148 |
-
image_tp += 1
|
| 149 |
-
else:
|
| 150 |
-
image_fn += 1
|
| 151 |
-
elif image_false_positives:
|
| 152 |
-
image_fp += 1
|
| 153 |
-
else:
|
| 154 |
-
image_tn += 1
|
| 155 |
-
|
| 156 |
-
tpr = image_tp / (image_tp + image_fn) if image_tp + image_fn else None
|
| 157 |
-
fpr = image_fp / (image_fp + image_tn) if image_fp + image_tn else None
|
| 158 |
-
accuracy = (image_tp + image_tn) / len(image_paths) if image_paths else None
|
| 159 |
-
precision = matched / (matched + false_positive_instances) if matched + false_positive_instances else None
|
| 160 |
-
recall = matched / gt_instances if gt_instances else None
|
| 161 |
-
mean_iou = sum(matched_ious) / len(matched_ious) if matched_ious else None
|
| 162 |
-
|
| 163 |
-
metrics = {
|
| 164 |
-
"weights": args.weights,
|
| 165 |
-
"data": args.data,
|
| 166 |
-
"split": args.split,
|
| 167 |
-
"images": len(image_paths),
|
| 168 |
-
"conf_threshold": args.conf,
|
| 169 |
-
"iou_threshold": args.iou,
|
| 170 |
-
"classification": {
|
| 171 |
-
"TPR": tpr,
|
| 172 |
-
"FPR": fpr,
|
| 173 |
-
"Accuracy": accuracy,
|
| 174 |
-
"TP": image_tp,
|
| 175 |
-
"FN": image_fn,
|
| 176 |
-
"FP": image_fp,
|
| 177 |
-
"TN": image_tn,
|
| 178 |
-
},
|
| 179 |
-
"detection": {
|
| 180 |
-
"gt_instances": gt_instances,
|
| 181 |
-
"pred_instances": pred_instances,
|
| 182 |
-
"matched_instances": matched,
|
| 183 |
-
"false_positives": false_positive_instances,
|
| 184 |
-
"false_negatives": false_negative_instances,
|
| 185 |
-
"precision": precision,
|
| 186 |
-
"recall": recall,
|
| 187 |
-
"mean_matched_iou": mean_iou,
|
| 188 |
-
},
|
| 189 |
-
"error_images": error_images,
|
| 190 |
-
}
|
| 191 |
-
output = Path(args.output)
|
| 192 |
-
output.parent.mkdir(parents=True, exist_ok=True)
|
| 193 |
-
output.write_text(json.dumps(metrics, indent=2, ensure_ascii=False), encoding="utf-8")
|
| 194 |
-
print(json.dumps(metrics, indent=2, ensure_ascii=False))
|
| 195 |
-
|
| 196 |
-
|
| 197 |
-
if __name__ == "__main__":
|
| 198 |
-
main()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|