cuibinge commited on
Commit
cfe4840
·
verified ·
1 Parent(s): 35a530e

Sync DETR-style dual backbone detector

Browse files
scripts/train_dual_backbone_detector.py CHANGED
@@ -1,4 +1,4 @@
1
- """Train a lightweight anchor-free detector on top of the dual DINOv3 backbone."""
2
 
3
  from __future__ import annotations
4
 
@@ -66,6 +66,13 @@ def parse_args() -> argparse.Namespace:
66
  parser.add_argument("--iou", type=float, default=0.5)
67
  parser.add_argument("--topk", type=int, default=300)
68
  parser.add_argument("--max-det", type=int, default=100)
 
 
 
 
 
 
 
69
  parser.add_argument("--seed", type=int, default=0)
70
  parser.add_argument("--no-pretrained", action="store_true")
71
  parser.add_argument("--data-parallel", action="store_true")
@@ -121,38 +128,105 @@ def collate(batch):
121
  return torch.stack(images), list(boxes), list(paths)
122
 
123
 
124
- class DenseBoxHead(nn.Module):
125
- def __init__(self, in_channels: int = 256, hidden: int = 256) -> None:
126
  super().__init__()
127
- self.shared = nn.Sequential(
128
- nn.Conv2d(in_channels, hidden, 3, padding=1),
129
- nn.GroupNorm(16, hidden),
130
- nn.GELU(),
131
- nn.Conv2d(hidden, hidden, 3, padding=1),
132
- nn.GroupNorm(16, hidden),
133
- nn.GELU(),
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
134
  )
135
- self.cls = nn.Conv2d(hidden, 1, 1)
136
- self.box = nn.Conv2d(hidden, 4, 1)
137
-
138
- def forward(self, x: Tensor) -> tuple[Tensor, Tensor]:
139
- feat = self.shared(x)
140
- return self.cls(feat), self.box(feat).sigmoid()
 
 
 
 
 
 
 
 
 
 
 
 
141
 
142
 
143
  class DualBackboneDetector(nn.Module):
144
- def __init__(self, vit_weights: str, convnext_weights: str, pretrained: bool = True) -> None:
 
 
 
 
 
 
 
 
145
  super().__init__()
146
  self.backbone = DINOv3ViTConvNeXtBackbone(
147
  vit_weights=vit_weights,
148
  convnext_weights=convnext_weights,
149
  pretrained=pretrained,
150
  )
151
- self.head = DenseBoxHead(in_channels=self.backbone.out_channels)
 
 
 
 
 
152
 
153
  def forward(self, x: Tensor) -> tuple[Tensor, Tensor, dict[str, Tensor]]:
154
  features = self.backbone(x)
155
- cls_logits, box_pred = self.head(features["detail"])
156
  return cls_logits, box_pred, features
157
 
158
 
@@ -202,27 +276,109 @@ def giou_loss(pred: Tensor, target: Tensor) -> Tensor:
202
  return (1 - giou).mean()
203
 
204
 
205
- def detection_loss(cls_logits: Tensor, box_pred: Tensor, boxes_list: list[Tensor]) -> tuple[Tensor, dict[str, float]]:
206
- _, _, h, w = cls_logits.shape
207
- cls_target, box_target, pos = make_targets(boxes_list, h, w, cls_logits.device)
208
- pos_count = pos.sum().clamp(min=1.0)
209
- cls_loss = F.binary_cross_entropy_with_logits(cls_logits, cls_target, reduction="none")
210
- neg_weight = torch.full_like(cls_loss, 0.05)
211
- weights = torch.where(pos > 0, torch.ones_like(cls_loss), neg_weight)
212
- cls_loss = (cls_loss * weights).sum() / (pos_count + neg_weight.sum())
213
- if pos.sum() > 0:
214
- pred_pos = box_pred.permute(0, 2, 3, 1)[pos[:, 0].bool()]
215
- target_pos = box_target.permute(0, 2, 3, 1)[pos[:, 0].bool()]
216
- xy1 = torch.minimum(pred_pos[:, :2], pred_pos[:, 2:])
217
- xy2 = torch.maximum(pred_pos[:, :2], pred_pos[:, 2:])
218
- pred_pos = torch.cat([xy1, xy2], dim=1)
219
- l1 = F.l1_loss(pred_pos, target_pos)
220
- giou = giou_loss(pred_pos, target_pos)
221
- box_loss = l1 + giou
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
222
  else:
223
- box_loss = box_pred.sum() * 0
224
- total = cls_loss + 5.0 * box_loss
225
- return total, {"cls_loss": float(cls_loss.detach()), "box_loss": float(box_loss.detach())}
 
 
 
 
 
 
 
226
 
227
 
228
  def nms(boxes: Tensor, scores: Tensor, iou_threshold: float, max_det: int) -> Tensor:
@@ -241,8 +397,8 @@ def nms(boxes: Tensor, scores: Tensor, iou_threshold: float, max_det: int) -> Te
241
 
242
 
243
  def decode_predictions(cls_logits: Tensor, box_pred: Tensor, conf: float, topk: int, nms_iou: float, max_det: int) -> list[Tensor]:
244
- scores = cls_logits.sigmoid().flatten(1)
245
- boxes = box_pred.permute(0, 2, 3, 1).reshape(cls_logits.shape[0], -1, 4)
246
  decoded = []
247
  for score_row, box_row in zip(scores, boxes):
248
  keep = score_row >= conf
@@ -356,7 +512,14 @@ def train() -> None:
356
  val_loader = DataLoader(val_ds, batch_size=args.batch, shuffle=False, num_workers=args.workers, collate_fn=collate)
357
  test_loader = DataLoader(test_ds, batch_size=args.batch, shuffle=False, num_workers=args.workers, collate_fn=collate)
358
 
359
- model = DualBackboneDetector(args.vit_weights, args.convnext_weights, pretrained=not args.no_pretrained).to(device)
 
 
 
 
 
 
 
360
  if args.data_parallel and torch.cuda.device_count() > 1:
361
  model = nn.DataParallel(model)
362
  raw = model.module if isinstance(model, nn.DataParallel) else model
@@ -372,30 +535,57 @@ def train() -> None:
372
  best_recall = -1.0
373
  history_path = output_dir / "history.csv"
374
  with history_path.open("w", newline="", encoding="utf-8") as fp:
375
- writer = csv.DictWriter(fp, fieldnames=["epoch", "loss", "cls_loss", "box_loss", "val_precision", "val_recall", "val_mean_iou"])
 
 
 
 
 
 
 
 
 
 
 
 
 
376
  writer.writeheader()
377
  for epoch in range(1, args.epochs + 1):
378
  raw.train()
379
  losses = []
380
  cls_losses = []
381
  box_losses = []
 
 
382
  for images, boxes_list, _ in train_loader:
383
  images = images.to(device)
384
  optimizer.zero_grad(set_to_none=True)
385
  cls_logits, box_pred, _ = model(images)
386
- loss, loss_items = detection_loss(cls_logits, box_pred, boxes_list)
 
 
 
 
 
 
 
 
387
  loss.backward()
388
  torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
389
  optimizer.step()
390
  losses.append(float(loss.detach()))
391
  cls_losses.append(loss_items["cls_loss"])
392
  box_losses.append(loss_items["box_loss"])
 
 
393
  val_metrics = evaluate(model, val_loader, device, args.conf, args.iou, args.topk, args.max_det)
394
  row = {
395
  "epoch": epoch,
396
  "loss": sum(losses) / max(len(losses), 1),
397
  "cls_loss": sum(cls_losses) / max(len(cls_losses), 1),
398
  "box_loss": sum(box_losses) / max(len(box_losses), 1),
 
 
399
  "val_precision": val_metrics.precision,
400
  "val_recall": val_metrics.recall,
401
  "val_mean_iou": val_metrics.mean_iou,
 
1
+ """Train a DETR-style detector on top of the dual DINOv3 backbone."""
2
 
3
  from __future__ import annotations
4
 
 
66
  parser.add_argument("--iou", type=float, default=0.5)
67
  parser.add_argument("--topk", type=int, default=300)
68
  parser.add_argument("--max-det", type=int, default=100)
69
+ parser.add_argument("--num-queries", type=int, default=160)
70
+ parser.add_argument("--decoder-layers", type=int, default=4)
71
+ parser.add_argument("--decoder-heads", type=int, default=8)
72
+ parser.add_argument("--match-class-cost", type=float, default=2.0)
73
+ parser.add_argument("--match-l1-cost", type=float, default=5.0)
74
+ parser.add_argument("--match-giou-cost", type=float, default=2.0)
75
+ parser.add_argument("--no-object-weight", type=float, default=0.1)
76
  parser.add_argument("--seed", type=int, default=0)
77
  parser.add_argument("--no-pretrained", action="store_true")
78
  parser.add_argument("--data-parallel", action="store_true")
 
128
  return torch.stack(images), list(boxes), list(paths)
129
 
130
 
131
+ class MLP(nn.Module):
132
+ def __init__(self, input_dim: int, hidden_dim: int, output_dim: int, num_layers: int) -> None:
133
  super().__init__()
134
+ layers = []
135
+ for idx in range(num_layers):
136
+ in_dim = input_dim if idx == 0 else hidden_dim
137
+ out_dim = output_dim if idx == num_layers - 1 else hidden_dim
138
+ layers.append(nn.Linear(in_dim, out_dim))
139
+ if idx < num_layers - 1:
140
+ layers.append(nn.GELU())
141
+ self.layers = nn.Sequential(*layers)
142
+
143
+ def forward(self, x: Tensor) -> Tensor:
144
+ return self.layers(x)
145
+
146
+
147
+ def sine_position_embedding(height: int, width: int, dim: int, device: torch.device, dtype: torch.dtype) -> Tensor:
148
+ if dim % 4 != 0:
149
+ raise ValueError("Position embedding dim must be divisible by 4")
150
+ y, x = torch.meshgrid(torch.arange(height, device=device), torch.arange(width, device=device), indexing="ij")
151
+ omega = torch.arange(dim // 4, device=device, dtype=dtype)
152
+ omega = 1.0 / (10000 ** (omega / max(dim // 4, 1)))
153
+ y = y.reshape(-1, 1).to(dtype) * omega
154
+ x = x.reshape(-1, 1).to(dtype) * omega
155
+ return torch.cat([torch.sin(y), torch.cos(y), torch.sin(x), torch.cos(x)], dim=1)
156
+
157
+
158
+ class SetPredictionHead(nn.Module):
159
+ """Small DETR-style query decoder over fused semantic and detail memories."""
160
+
161
+ def __init__(
162
+ self,
163
+ in_channels: int = 256,
164
+ hidden: int = 256,
165
+ num_queries: int = 160,
166
+ decoder_layers: int = 4,
167
+ decoder_heads: int = 8,
168
+ ) -> None:
169
+ super().__init__()
170
+ self.hidden = hidden
171
+ self.fused_proj = nn.Conv2d(in_channels, hidden, kernel_size=1)
172
+ self.detail_proj = nn.Conv2d(in_channels, hidden, kernel_size=1)
173
+ self.level_embed = nn.Parameter(torch.zeros(2, hidden))
174
+ self.query_embed = nn.Embedding(num_queries, hidden)
175
+ layer = nn.TransformerDecoderLayer(
176
+ d_model=hidden,
177
+ nhead=decoder_heads,
178
+ dim_feedforward=hidden * 4,
179
+ dropout=0.0,
180
+ activation="gelu",
181
+ batch_first=True,
182
+ norm_first=True,
183
  )
184
+ self.decoder = nn.TransformerDecoder(layer, num_layers=decoder_layers)
185
+ self.class_embed = nn.Linear(hidden, 1)
186
+ self.box_embed = MLP(hidden, hidden, 4, num_layers=3)
187
+ nn.init.constant_(self.class_embed.bias, -2.0)
188
+
189
+ def _flatten_level(self, feat: Tensor, level: int) -> Tensor:
190
+ b, c, h, w = feat.shape
191
+ pos = sine_position_embedding(h, w, c, feat.device, feat.dtype)
192
+ tokens = feat.flatten(2).transpose(1, 2).contiguous()
193
+ return tokens + pos.unsqueeze(0) + self.level_embed[level].to(dtype=feat.dtype).view(1, 1, -1)
194
+
195
+ def forward(self, features: dict[str, Tensor]) -> tuple[Tensor, Tensor]:
196
+ fused = self.fused_proj(features["fused"])
197
+ detail = self.detail_proj(features["detail"])
198
+ memory = torch.cat([self._flatten_level(fused, 0), self._flatten_level(detail, 1)], dim=1)
199
+ queries = self.query_embed.weight.unsqueeze(0).expand(memory.shape[0], -1, -1)
200
+ decoded = self.decoder(tgt=queries, memory=memory)
201
+ return self.class_embed(decoded).squeeze(-1), self.box_embed(decoded).sigmoid()
202
 
203
 
204
  class DualBackboneDetector(nn.Module):
205
+ def __init__(
206
+ self,
207
+ vit_weights: str,
208
+ convnext_weights: str,
209
+ pretrained: bool = True,
210
+ num_queries: int = 160,
211
+ decoder_layers: int = 4,
212
+ decoder_heads: int = 8,
213
+ ) -> None:
214
  super().__init__()
215
  self.backbone = DINOv3ViTConvNeXtBackbone(
216
  vit_weights=vit_weights,
217
  convnext_weights=convnext_weights,
218
  pretrained=pretrained,
219
  )
220
+ self.head = SetPredictionHead(
221
+ in_channels=self.backbone.out_channels,
222
+ num_queries=num_queries,
223
+ decoder_layers=decoder_layers,
224
+ decoder_heads=decoder_heads,
225
+ )
226
 
227
  def forward(self, x: Tensor) -> tuple[Tensor, Tensor, dict[str, Tensor]]:
228
  features = self.backbone(x)
229
+ cls_logits, box_pred = self.head(features)
230
  return cls_logits, box_pred, features
231
 
232
 
 
276
  return (1 - giou).mean()
277
 
278
 
279
+ def linear_sum_assignment_torch(cost: Tensor) -> tuple[Tensor, Tensor]:
280
+ try:
281
+ from scipy.optimize import linear_sum_assignment
282
+
283
+ rows, cols = linear_sum_assignment(cost.detach().cpu().numpy())
284
+ return (
285
+ torch.as_tensor(rows, dtype=torch.long, device=cost.device),
286
+ torch.as_tensor(cols, dtype=torch.long, device=cost.device),
287
+ )
288
+ except Exception:
289
+ # Greedy fallback keeps training dependency-light. SciPy is preferred when available.
290
+ rows = []
291
+ cols = []
292
+ used_rows: set[int] = set()
293
+ used_cols: set[int] = set()
294
+ flat = cost.flatten().argsort()
295
+ n_cols = cost.shape[1]
296
+ for item in flat:
297
+ row = int(item // n_cols)
298
+ col = int(item % n_cols)
299
+ if row in used_rows or col in used_cols:
300
+ continue
301
+ used_rows.add(row)
302
+ used_cols.add(col)
303
+ rows.append(row)
304
+ cols.append(col)
305
+ if len(cols) == min(cost.shape):
306
+ break
307
+ return (
308
+ torch.as_tensor(rows, dtype=torch.long, device=cost.device),
309
+ torch.as_tensor(cols, dtype=torch.long, device=cost.device),
310
+ )
311
+
312
+
313
+ def generalized_iou_matrix(boxes1: Tensor, boxes2: Tensor) -> Tensor:
314
+ iou = box_iou_matrix(boxes1, boxes2)
315
+ if boxes1.numel() == 0 or boxes2.numel() == 0:
316
+ return iou
317
+ lt_c = torch.minimum(boxes1[:, None, :2], boxes2[None, :, :2])
318
+ rb_c = torch.maximum(boxes1[:, None, 2:], boxes2[None, :, 2:])
319
+ wh_c = (rb_c - lt_c).clamp(min=0)
320
+ area_c = (wh_c[..., 0] * wh_c[..., 1]).clamp(min=1e-8)
321
+ area1 = ((boxes1[:, 2] - boxes1[:, 0]).clamp(min=0) * (boxes1[:, 3] - boxes1[:, 1]).clamp(min=0))[:, None]
322
+ area2 = ((boxes2[:, 2] - boxes2[:, 0]).clamp(min=0) * (boxes2[:, 3] - boxes2[:, 1]).clamp(min=0))[None, :]
323
+ lt = torch.maximum(boxes1[:, None, :2], boxes2[None, :, :2])
324
+ rb = torch.minimum(boxes1[:, None, 2:], boxes2[None, :, 2:])
325
+ wh = (rb - lt).clamp(min=0)
326
+ inter = wh[..., 0] * wh[..., 1]
327
+ union = (area1 + area2 - inter).clamp(min=1e-8)
328
+ return iou - (area_c - union) / area_c
329
+
330
+
331
+ def set_prediction_loss(
332
+ pred_logits: Tensor,
333
+ pred_boxes: Tensor,
334
+ boxes_list: list[Tensor],
335
+ class_cost: float,
336
+ l1_cost: float,
337
+ giou_cost: float,
338
+ no_object_weight: float,
339
+ ) -> tuple[Tensor, dict[str, float]]:
340
+ cls_targets = torch.zeros_like(pred_logits)
341
+ matched_pred_boxes = []
342
+ matched_target_boxes = []
343
+ total_targets = 0
344
+
345
+ for batch_idx, target_boxes_cpu in enumerate(boxes_list):
346
+ target_boxes = target_boxes_cpu.to(pred_boxes.device)
347
+ total_targets += int(target_boxes.shape[0])
348
+ if target_boxes.numel() == 0:
349
+ continue
350
+ prob = pred_logits[batch_idx].sigmoid()
351
+ class_match_cost = -prob[:, None].expand(-1, target_boxes.shape[0])
352
+ l1_match_cost = torch.cdist(pred_boxes[batch_idx], target_boxes, p=1)
353
+ giou_match_cost = -generalized_iou_matrix(pred_boxes[batch_idx], target_boxes)
354
+ cost = class_cost * class_match_cost + l1_cost * l1_match_cost + giou_cost * giou_match_cost
355
+ pred_idx, target_idx = linear_sum_assignment_torch(cost)
356
+ cls_targets[batch_idx, pred_idx] = 1.0
357
+ matched_pred_boxes.append(pred_boxes[batch_idx, pred_idx])
358
+ matched_target_boxes.append(target_boxes[target_idx])
359
+
360
+ cls_weight = torch.full_like(cls_targets, no_object_weight)
361
+ cls_weight = torch.where(cls_targets > 0, torch.ones_like(cls_weight), cls_weight)
362
+ cls_loss = F.binary_cross_entropy_with_logits(pred_logits, cls_targets, weight=cls_weight, reduction="sum")
363
+ cls_loss = cls_loss / max(total_targets, 1)
364
+
365
+ if matched_pred_boxes:
366
+ pred_cat = torch.cat(matched_pred_boxes, dim=0)
367
+ target_cat = torch.cat(matched_target_boxes, dim=0)
368
+ l1_loss = F.l1_loss(pred_cat, target_cat, reduction="none").sum() / max(total_targets, 1)
369
+ giou = generalized_iou_matrix(pred_cat, target_cat).diag()
370
+ giou_loss_value = (1.0 - giou).sum() / max(total_targets, 1)
371
  else:
372
+ l1_loss = pred_boxes.sum() * 0
373
+ giou_loss_value = pred_boxes.sum() * 0
374
+
375
+ total = cls_loss + l1_cost * l1_loss + giou_cost * giou_loss_value
376
+ return total, {
377
+ "cls_loss": float(cls_loss.detach()),
378
+ "box_loss": float((l1_loss + giou_loss_value).detach()),
379
+ "l1_loss": float(l1_loss.detach()),
380
+ "giou_loss": float(giou_loss_value.detach()),
381
+ }
382
 
383
 
384
  def nms(boxes: Tensor, scores: Tensor, iou_threshold: float, max_det: int) -> Tensor:
 
397
 
398
 
399
  def decode_predictions(cls_logits: Tensor, box_pred: Tensor, conf: float, topk: int, nms_iou: float, max_det: int) -> list[Tensor]:
400
+ scores = cls_logits.sigmoid()
401
+ boxes = box_pred
402
  decoded = []
403
  for score_row, box_row in zip(scores, boxes):
404
  keep = score_row >= conf
 
512
  val_loader = DataLoader(val_ds, batch_size=args.batch, shuffle=False, num_workers=args.workers, collate_fn=collate)
513
  test_loader = DataLoader(test_ds, batch_size=args.batch, shuffle=False, num_workers=args.workers, collate_fn=collate)
514
 
515
+ model = DualBackboneDetector(
516
+ args.vit_weights,
517
+ args.convnext_weights,
518
+ pretrained=not args.no_pretrained,
519
+ num_queries=args.num_queries,
520
+ decoder_layers=args.decoder_layers,
521
+ decoder_heads=args.decoder_heads,
522
+ ).to(device)
523
  if args.data_parallel and torch.cuda.device_count() > 1:
524
  model = nn.DataParallel(model)
525
  raw = model.module if isinstance(model, nn.DataParallel) else model
 
535
  best_recall = -1.0
536
  history_path = output_dir / "history.csv"
537
  with history_path.open("w", newline="", encoding="utf-8") as fp:
538
+ writer = csv.DictWriter(
539
+ fp,
540
+ fieldnames=[
541
+ "epoch",
542
+ "loss",
543
+ "cls_loss",
544
+ "box_loss",
545
+ "l1_loss",
546
+ "giou_loss",
547
+ "val_precision",
548
+ "val_recall",
549
+ "val_mean_iou",
550
+ ],
551
+ )
552
  writer.writeheader()
553
  for epoch in range(1, args.epochs + 1):
554
  raw.train()
555
  losses = []
556
  cls_losses = []
557
  box_losses = []
558
+ l1_losses = []
559
+ giou_losses = []
560
  for images, boxes_list, _ in train_loader:
561
  images = images.to(device)
562
  optimizer.zero_grad(set_to_none=True)
563
  cls_logits, box_pred, _ = model(images)
564
+ loss, loss_items = set_prediction_loss(
565
+ cls_logits,
566
+ box_pred,
567
+ boxes_list,
568
+ class_cost=args.match_class_cost,
569
+ l1_cost=args.match_l1_cost,
570
+ giou_cost=args.match_giou_cost,
571
+ no_object_weight=args.no_object_weight,
572
+ )
573
  loss.backward()
574
  torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
575
  optimizer.step()
576
  losses.append(float(loss.detach()))
577
  cls_losses.append(loss_items["cls_loss"])
578
  box_losses.append(loss_items["box_loss"])
579
+ l1_losses.append(loss_items["l1_loss"])
580
+ giou_losses.append(loss_items["giou_loss"])
581
  val_metrics = evaluate(model, val_loader, device, args.conf, args.iou, args.topk, args.max_det)
582
  row = {
583
  "epoch": epoch,
584
  "loss": sum(losses) / max(len(losses), 1),
585
  "cls_loss": sum(cls_losses) / max(len(cls_losses), 1),
586
  "box_loss": sum(box_losses) / max(len(box_losses), 1),
587
+ "l1_loss": sum(l1_losses) / max(len(l1_losses), 1),
588
+ "giou_loss": sum(giou_losses) / max(len(giou_losses), 1),
589
  "val_precision": val_metrics.precision,
590
  "val_recall": val_metrics.recall,
591
  "val_mean_iou": val_metrics.mean_iou,