cuibinge commited on
Commit
35a530e
·
verified ·
1 Parent(s): e88d9ee

Sync dual backbone detector training script

Browse files
scripts/train_dual_backbone_detector.py ADDED
@@ -0,0 +1,419 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Train a lightweight anchor-free detector on top of the dual DINOv3 backbone."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import argparse
6
+ import csv
7
+ import json
8
+ import math
9
+ import random
10
+ import sys
11
+ from dataclasses import asdict, dataclass
12
+ from pathlib import Path
13
+
14
+ import torch
15
+ from PIL import Image
16
+ from torch import Tensor, nn
17
+ import torch.nn.functional as F
18
+ from torch.utils.data import DataLoader, Dataset
19
+ from torchvision.transforms import functional as TF
20
+
21
+ ROOT = Path(__file__).resolve().parents[1]
22
+ if str(ROOT) not in sys.path:
23
+ sys.path.append(str(ROOT))
24
+
25
+ from marine_dual_dinov3_backbone import DINOv3ViTConvNeXtBackbone # noqa: E402
26
+
27
+
28
+ IMAGE_SUFFIXES = {".jpg", ".jpeg", ".png", ".tif", ".tiff"}
29
+
30
+
31
+ @dataclass
32
+ class Metrics:
33
+ images: int
34
+ gt_instances: int
35
+ pred_instances: int
36
+ matched_instances: int
37
+ false_positives: int
38
+ false_negatives: int
39
+ precision: float
40
+ recall: float
41
+ mean_iou: float
42
+ image_tp: int
43
+ image_fn: int
44
+ image_fp: int
45
+ image_tn: int
46
+ tpr: float | None
47
+ fpr: float | None
48
+ accuracy: float | None
49
+
50
+
51
+ def parse_args() -> argparse.Namespace:
52
+ parser = argparse.ArgumentParser(description=__doc__)
53
+ parser.add_argument("--data-root", required=True)
54
+ parser.add_argument("--vit-weights", required=True)
55
+ parser.add_argument("--convnext-weights", required=True)
56
+ parser.add_argument("--output-dir", required=True)
57
+ parser.add_argument("--epochs", type=int, default=20)
58
+ parser.add_argument("--imgsz", type=int, default=224)
59
+ parser.add_argument("--batch", type=int, default=1)
60
+ parser.add_argument("--workers", type=int, default=0)
61
+ parser.add_argument("--device", default="cuda")
62
+ parser.add_argument("--lr", type=float, default=1e-4)
63
+ parser.add_argument("--backbone-lr", type=float, default=1e-5)
64
+ parser.add_argument("--weight-decay", type=float, default=1e-4)
65
+ parser.add_argument("--conf", type=float, default=0.25)
66
+ parser.add_argument("--iou", type=float, default=0.5)
67
+ parser.add_argument("--topk", type=int, default=300)
68
+ parser.add_argument("--max-det", type=int, default=100)
69
+ parser.add_argument("--seed", type=int, default=0)
70
+ parser.add_argument("--no-pretrained", action="store_true")
71
+ parser.add_argument("--data-parallel", action="store_true")
72
+ return parser.parse_args()
73
+
74
+
75
+ def image_paths_for_split(root: Path, split: str) -> list[Path]:
76
+ image_dir = root / "images" / split
77
+ return sorted(path for path in image_dir.iterdir() if path.suffix.lower() in IMAGE_SUFFIXES)
78
+
79
+
80
+ def label_path_for_image(root: Path, image_path: Path, split: str) -> Path:
81
+ return root / "labels" / split / f"{image_path.stem}.txt"
82
+
83
+
84
+ def load_yolo_boxes(label_path: Path) -> Tensor:
85
+ boxes = []
86
+ if label_path.exists():
87
+ for line in label_path.read_text(encoding="utf-8").splitlines():
88
+ if not line.strip():
89
+ continue
90
+ _, cx, cy, w, h = [float(part) for part in line.split()[:5]]
91
+ x1 = max(0.0, cx - w / 2)
92
+ y1 = max(0.0, cy - h / 2)
93
+ x2 = min(1.0, cx + w / 2)
94
+ y2 = min(1.0, cy + h / 2)
95
+ if x2 > x1 and y2 > y1:
96
+ boxes.append([x1, y1, x2, y2])
97
+ return torch.tensor(boxes, dtype=torch.float32)
98
+
99
+
100
+ class YoloDetectionDataset(Dataset):
101
+ def __init__(self, root: str | Path, split: str, image_size: int) -> None:
102
+ self.root = Path(root)
103
+ self.split = split
104
+ self.image_size = image_size
105
+ self.images = image_paths_for_split(self.root, split)
106
+
107
+ def __len__(self) -> int:
108
+ return len(self.images)
109
+
110
+ def __getitem__(self, idx: int) -> tuple[Tensor, Tensor, str]:
111
+ image_path = self.images[idx]
112
+ image = Image.open(image_path).convert("RGB")
113
+ image = image.resize((self.image_size, self.image_size), Image.BILINEAR)
114
+ tensor = TF.to_tensor(image)
115
+ boxes = load_yolo_boxes(label_path_for_image(self.root, image_path, self.split))
116
+ return tensor, boxes, str(image_path)
117
+
118
+
119
+ def collate(batch):
120
+ images, boxes, paths = zip(*batch)
121
+ return torch.stack(images), list(boxes), list(paths)
122
+
123
+
124
+ class DenseBoxHead(nn.Module):
125
+ def __init__(self, in_channels: int = 256, hidden: int = 256) -> None:
126
+ super().__init__()
127
+ self.shared = nn.Sequential(
128
+ nn.Conv2d(in_channels, hidden, 3, padding=1),
129
+ nn.GroupNorm(16, hidden),
130
+ nn.GELU(),
131
+ nn.Conv2d(hidden, hidden, 3, padding=1),
132
+ nn.GroupNorm(16, hidden),
133
+ nn.GELU(),
134
+ )
135
+ self.cls = nn.Conv2d(hidden, 1, 1)
136
+ self.box = nn.Conv2d(hidden, 4, 1)
137
+
138
+ def forward(self, x: Tensor) -> tuple[Tensor, Tensor]:
139
+ feat = self.shared(x)
140
+ return self.cls(feat), self.box(feat).sigmoid()
141
+
142
+
143
+ class DualBackboneDetector(nn.Module):
144
+ def __init__(self, vit_weights: str, convnext_weights: str, pretrained: bool = True) -> None:
145
+ super().__init__()
146
+ self.backbone = DINOv3ViTConvNeXtBackbone(
147
+ vit_weights=vit_weights,
148
+ convnext_weights=convnext_weights,
149
+ pretrained=pretrained,
150
+ )
151
+ self.head = DenseBoxHead(in_channels=self.backbone.out_channels)
152
+
153
+ def forward(self, x: Tensor) -> tuple[Tensor, Tensor, dict[str, Tensor]]:
154
+ features = self.backbone(x)
155
+ cls_logits, box_pred = self.head(features["detail"])
156
+ return cls_logits, box_pred, features
157
+
158
+
159
+ def make_targets(boxes_list: list[Tensor], height: int, width: int, device: torch.device) -> tuple[Tensor, Tensor, Tensor]:
160
+ cls = torch.zeros((len(boxes_list), 1, height, width), device=device)
161
+ box = torch.zeros((len(boxes_list), 4, height, width), device=device)
162
+ pos = torch.zeros((len(boxes_list), 1, height, width), device=device)
163
+ for b, boxes in enumerate(boxes_list):
164
+ boxes = boxes.to(device)
165
+ for target_box in boxes:
166
+ cx = float((target_box[0] + target_box[2]) / 2)
167
+ cy = float((target_box[1] + target_box[3]) / 2)
168
+ gx = min(width - 1, max(0, int(cx * width)))
169
+ gy = min(height - 1, max(0, int(cy * height)))
170
+ cls[b, 0, gy, gx] = 1.0
171
+ box[b, :, gy, gx] = target_box
172
+ pos[b, 0, gy, gx] = 1.0
173
+ return cls, box, pos
174
+
175
+
176
+ def box_iou_matrix(a: Tensor, b: Tensor) -> Tensor:
177
+ if a.numel() == 0 or b.numel() == 0:
178
+ return a.new_zeros((a.shape[0], b.shape[0]))
179
+ lt = torch.maximum(a[:, None, :2], b[None, :, :2])
180
+ rb = torch.minimum(a[:, None, 2:], b[None, :, 2:])
181
+ wh = (rb - lt).clamp(min=0)
182
+ inter = wh[..., 0] * wh[..., 1]
183
+ area_a = ((a[:, 2] - a[:, 0]).clamp(min=0) * (a[:, 3] - a[:, 1]).clamp(min=0))[:, None]
184
+ area_b = ((b[:, 2] - b[:, 0]).clamp(min=0) * (b[:, 3] - b[:, 1]).clamp(min=0))[None, :]
185
+ return inter / (area_a + area_b - inter).clamp(min=1e-8)
186
+
187
+
188
+ def giou_loss(pred: Tensor, target: Tensor) -> Tensor:
189
+ iou = box_iou_matrix(pred, target).diag()
190
+ lt_c = torch.minimum(pred[:, :2], target[:, :2])
191
+ rb_c = torch.maximum(pred[:, 2:], target[:, 2:])
192
+ wh_c = (rb_c - lt_c).clamp(min=0)
193
+ area_c = (wh_c[:, 0] * wh_c[:, 1]).clamp(min=1e-8)
194
+ area_p = (pred[:, 2] - pred[:, 0]).clamp(min=0) * (pred[:, 3] - pred[:, 1]).clamp(min=0)
195
+ area_t = (target[:, 2] - target[:, 0]).clamp(min=0) * (target[:, 3] - target[:, 1]).clamp(min=0)
196
+ lt = torch.maximum(pred[:, :2], target[:, :2])
197
+ rb = torch.minimum(pred[:, 2:], target[:, 2:])
198
+ wh = (rb - lt).clamp(min=0)
199
+ inter = wh[:, 0] * wh[:, 1]
200
+ union = (area_p + area_t - inter).clamp(min=1e-8)
201
+ giou = iou - (area_c - union) / area_c
202
+ return (1 - giou).mean()
203
+
204
+
205
+ def detection_loss(cls_logits: Tensor, box_pred: Tensor, boxes_list: list[Tensor]) -> tuple[Tensor, dict[str, float]]:
206
+ _, _, h, w = cls_logits.shape
207
+ cls_target, box_target, pos = make_targets(boxes_list, h, w, cls_logits.device)
208
+ pos_count = pos.sum().clamp(min=1.0)
209
+ cls_loss = F.binary_cross_entropy_with_logits(cls_logits, cls_target, reduction="none")
210
+ neg_weight = torch.full_like(cls_loss, 0.05)
211
+ weights = torch.where(pos > 0, torch.ones_like(cls_loss), neg_weight)
212
+ cls_loss = (cls_loss * weights).sum() / (pos_count + neg_weight.sum())
213
+ if pos.sum() > 0:
214
+ pred_pos = box_pred.permute(0, 2, 3, 1)[pos[:, 0].bool()]
215
+ target_pos = box_target.permute(0, 2, 3, 1)[pos[:, 0].bool()]
216
+ xy1 = torch.minimum(pred_pos[:, :2], pred_pos[:, 2:])
217
+ xy2 = torch.maximum(pred_pos[:, :2], pred_pos[:, 2:])
218
+ pred_pos = torch.cat([xy1, xy2], dim=1)
219
+ l1 = F.l1_loss(pred_pos, target_pos)
220
+ giou = giou_loss(pred_pos, target_pos)
221
+ box_loss = l1 + giou
222
+ else:
223
+ box_loss = box_pred.sum() * 0
224
+ total = cls_loss + 5.0 * box_loss
225
+ return total, {"cls_loss": float(cls_loss.detach()), "box_loss": float(box_loss.detach())}
226
+
227
+
228
+ def nms(boxes: Tensor, scores: Tensor, iou_threshold: float, max_det: int) -> Tensor:
229
+ if boxes.numel() == 0:
230
+ return torch.empty(0, dtype=torch.long, device=boxes.device)
231
+ order = scores.argsort(descending=True)
232
+ keep = []
233
+ while order.numel() > 0 and len(keep) < max_det:
234
+ idx = order[0]
235
+ keep.append(idx)
236
+ if order.numel() == 1:
237
+ break
238
+ ious = box_iou_matrix(boxes[idx][None], boxes[order[1:]])[0]
239
+ order = order[1:][ious <= iou_threshold]
240
+ return torch.stack(keep) if keep else torch.empty(0, dtype=torch.long, device=boxes.device)
241
+
242
+
243
+ def decode_predictions(cls_logits: Tensor, box_pred: Tensor, conf: float, topk: int, nms_iou: float, max_det: int) -> list[Tensor]:
244
+ scores = cls_logits.sigmoid().flatten(1)
245
+ boxes = box_pred.permute(0, 2, 3, 1).reshape(cls_logits.shape[0], -1, 4)
246
+ decoded = []
247
+ for score_row, box_row in zip(scores, boxes):
248
+ keep = score_row >= conf
249
+ if keep.sum() == 0:
250
+ decoded.append(box_row.new_zeros((0, 5)))
251
+ continue
252
+ kept_scores = score_row[keep]
253
+ kept_boxes = box_row[keep]
254
+ if kept_scores.numel() > topk:
255
+ kept_scores, idx = kept_scores.topk(topk)
256
+ kept_boxes = kept_boxes[idx]
257
+ xy1 = torch.minimum(kept_boxes[:, :2], kept_boxes[:, 2:])
258
+ xy2 = torch.maximum(kept_boxes[:, :2], kept_boxes[:, 2:])
259
+ kept_boxes = torch.cat([xy1, xy2], dim=1).clamp(0, 1)
260
+ keep_idx = nms(kept_boxes, kept_scores, nms_iou, max_det)
261
+ kept_boxes = kept_boxes[keep_idx]
262
+ kept_scores = kept_scores[keep_idx]
263
+ decoded.append(torch.cat([kept_boxes, kept_scores[:, None]], dim=1))
264
+ return decoded
265
+
266
+
267
+ def evaluate(
268
+ model: nn.Module,
269
+ loader: DataLoader,
270
+ device: torch.device,
271
+ conf: float,
272
+ iou_threshold: float,
273
+ topk: int,
274
+ max_det: int,
275
+ ) -> Metrics:
276
+ module = model.module if isinstance(model, nn.DataParallel) else model
277
+ module.eval()
278
+ matched = fp = fn = gt_total = pred_total = image_tp = image_fn = image_fp = image_tn = 0
279
+ matched_ious: list[float] = []
280
+ with torch.no_grad():
281
+ for images, boxes_list, _ in loader:
282
+ images = images.to(device)
283
+ cls_logits, box_pred, _ = model(images)
284
+ preds = decode_predictions(cls_logits, box_pred, conf=conf, topk=topk, nms_iou=iou_threshold, max_det=max_det)
285
+ for pred, gt in zip(preds, boxes_list):
286
+ gt = gt.to(device)
287
+ pred_boxes = pred[:, :4]
288
+ gt_total += int(gt.shape[0])
289
+ pred_total += int(pred_boxes.shape[0])
290
+ used = set()
291
+ image_match = 0
292
+ image_fp_count = 0
293
+ for pbox in pred_boxes:
294
+ if gt.numel() == 0:
295
+ fp += 1
296
+ image_fp_count += 1
297
+ continue
298
+ ious = box_iou_matrix(pbox[None], gt)[0]
299
+ best_iou, best_idx = ious.max(dim=0)
300
+ if float(best_iou) >= iou_threshold and int(best_idx) not in used:
301
+ used.add(int(best_idx))
302
+ matched += 1
303
+ image_match += 1
304
+ matched_ious.append(float(best_iou))
305
+ else:
306
+ fp += 1
307
+ image_fp_count += 1
308
+ missed = int(gt.shape[0]) - image_match
309
+ fn += missed
310
+ if gt.shape[0] > 0:
311
+ if image_match:
312
+ image_tp += 1
313
+ else:
314
+ image_fn += 1
315
+ elif image_fp_count:
316
+ image_fp += 1
317
+ else:
318
+ image_tn += 1
319
+ precision = matched / (matched + fp) if matched + fp else 0.0
320
+ recall = matched / gt_total if gt_total else 0.0
321
+ mean_iou = sum(matched_ious) / len(matched_ious) if matched_ious else 0.0
322
+ tpr = image_tp / (image_tp + image_fn) if image_tp + image_fn else None
323
+ fpr = image_fp / (image_fp + image_tn) if image_fp + image_tn else None
324
+ accuracy = (image_tp + image_tn) / (image_tp + image_fn + image_fp + image_tn) if image_tp + image_fn + image_fp + image_tn else None
325
+ return Metrics(
326
+ images=len(loader.dataset),
327
+ gt_instances=gt_total,
328
+ pred_instances=pred_total,
329
+ matched_instances=matched,
330
+ false_positives=fp,
331
+ false_negatives=fn,
332
+ precision=precision,
333
+ recall=recall,
334
+ mean_iou=mean_iou,
335
+ image_tp=image_tp,
336
+ image_fn=image_fn,
337
+ image_fp=image_fp,
338
+ image_tn=image_tn,
339
+ tpr=tpr,
340
+ fpr=fpr,
341
+ accuracy=accuracy,
342
+ )
343
+
344
+
345
+ def train() -> None:
346
+ args = parse_args()
347
+ random.seed(args.seed)
348
+ torch.manual_seed(args.seed)
349
+ output_dir = Path(args.output_dir)
350
+ output_dir.mkdir(parents=True, exist_ok=True)
351
+ device = torch.device(args.device if torch.cuda.is_available() else "cpu")
352
+ train_ds = YoloDetectionDataset(args.data_root, "train", args.imgsz)
353
+ val_ds = YoloDetectionDataset(args.data_root, "val", args.imgsz)
354
+ test_ds = YoloDetectionDataset(args.data_root, "test", args.imgsz)
355
+ train_loader = DataLoader(train_ds, batch_size=args.batch, shuffle=True, num_workers=args.workers, collate_fn=collate)
356
+ val_loader = DataLoader(val_ds, batch_size=args.batch, shuffle=False, num_workers=args.workers, collate_fn=collate)
357
+ test_loader = DataLoader(test_ds, batch_size=args.batch, shuffle=False, num_workers=args.workers, collate_fn=collate)
358
+
359
+ model = DualBackboneDetector(args.vit_weights, args.convnext_weights, pretrained=not args.no_pretrained).to(device)
360
+ if args.data_parallel and torch.cuda.device_count() > 1:
361
+ model = nn.DataParallel(model)
362
+ raw = model.module if isinstance(model, nn.DataParallel) else model
363
+ backbone_params = [p for n, p in raw.backbone.named_parameters() if p.requires_grad]
364
+ head_params = [p for p in raw.head.parameters() if p.requires_grad]
365
+ optimizer = torch.optim.AdamW(
366
+ [
367
+ {"params": backbone_params, "lr": args.backbone_lr},
368
+ {"params": head_params, "lr": args.lr},
369
+ ],
370
+ weight_decay=args.weight_decay,
371
+ )
372
+ best_recall = -1.0
373
+ history_path = output_dir / "history.csv"
374
+ with history_path.open("w", newline="", encoding="utf-8") as fp:
375
+ writer = csv.DictWriter(fp, fieldnames=["epoch", "loss", "cls_loss", "box_loss", "val_precision", "val_recall", "val_mean_iou"])
376
+ writer.writeheader()
377
+ for epoch in range(1, args.epochs + 1):
378
+ raw.train()
379
+ losses = []
380
+ cls_losses = []
381
+ box_losses = []
382
+ for images, boxes_list, _ in train_loader:
383
+ images = images.to(device)
384
+ optimizer.zero_grad(set_to_none=True)
385
+ cls_logits, box_pred, _ = model(images)
386
+ loss, loss_items = detection_loss(cls_logits, box_pred, boxes_list)
387
+ loss.backward()
388
+ torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
389
+ optimizer.step()
390
+ losses.append(float(loss.detach()))
391
+ cls_losses.append(loss_items["cls_loss"])
392
+ box_losses.append(loss_items["box_loss"])
393
+ val_metrics = evaluate(model, val_loader, device, args.conf, args.iou, args.topk, args.max_det)
394
+ row = {
395
+ "epoch": epoch,
396
+ "loss": sum(losses) / max(len(losses), 1),
397
+ "cls_loss": sum(cls_losses) / max(len(cls_losses), 1),
398
+ "box_loss": sum(box_losses) / max(len(box_losses), 1),
399
+ "val_precision": val_metrics.precision,
400
+ "val_recall": val_metrics.recall,
401
+ "val_mean_iou": val_metrics.mean_iou,
402
+ }
403
+ writer.writerow(row)
404
+ fp.flush()
405
+ print(json.dumps(row), flush=True)
406
+ if val_metrics.recall > best_recall:
407
+ best_recall = val_metrics.recall
408
+ torch.save({"model": raw.state_dict(), "args": vars(args), "val_metrics": asdict(val_metrics)}, output_dir / "best.pt")
409
+ torch.save({"model": raw.state_dict(), "args": vars(args), "val_metrics": asdict(val_metrics)}, output_dir / "last.pt")
410
+
411
+ best = torch.load(output_dir / "best.pt", map_location=device)
412
+ raw.load_state_dict(best["model"])
413
+ test_metrics = evaluate(model, test_loader, device, args.conf, args.iou, args.topk, args.max_det)
414
+ (output_dir / "test_metrics.json").write_text(json.dumps(asdict(test_metrics), indent=2), encoding="utf-8")
415
+ print(json.dumps({"test": asdict(test_metrics)}, indent=2), flush=True)
416
+
417
+
418
+ if __name__ == "__main__":
419
+ train()