Sync DETR-style dual backbone detector
Browse files
scripts/train_dual_backbone_detector.py
CHANGED
|
@@ -1,4 +1,4 @@
|
|
| 1 |
-
"""Train a
|
| 2 |
|
| 3 |
from __future__ import annotations
|
| 4 |
|
|
@@ -66,6 +66,13 @@ def parse_args() -> argparse.Namespace:
|
|
| 66 |
parser.add_argument("--iou", type=float, default=0.5)
|
| 67 |
parser.add_argument("--topk", type=int, default=300)
|
| 68 |
parser.add_argument("--max-det", type=int, default=100)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 69 |
parser.add_argument("--seed", type=int, default=0)
|
| 70 |
parser.add_argument("--no-pretrained", action="store_true")
|
| 71 |
parser.add_argument("--data-parallel", action="store_true")
|
|
@@ -121,38 +128,105 @@ def collate(batch):
|
|
| 121 |
return torch.stack(images), list(boxes), list(paths)
|
| 122 |
|
| 123 |
|
| 124 |
-
class
|
| 125 |
-
def __init__(self,
|
| 126 |
super().__init__()
|
| 127 |
-
|
| 128 |
-
|
| 129 |
-
|
| 130 |
-
|
| 131 |
-
nn.
|
| 132 |
-
|
| 133 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 134 |
)
|
| 135 |
-
self.
|
| 136 |
-
self.
|
| 137 |
-
|
| 138 |
-
|
| 139 |
-
|
| 140 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 141 |
|
| 142 |
|
| 143 |
class DualBackboneDetector(nn.Module):
|
| 144 |
-
def __init__(
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 145 |
super().__init__()
|
| 146 |
self.backbone = DINOv3ViTConvNeXtBackbone(
|
| 147 |
vit_weights=vit_weights,
|
| 148 |
convnext_weights=convnext_weights,
|
| 149 |
pretrained=pretrained,
|
| 150 |
)
|
| 151 |
-
self.head =
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 152 |
|
| 153 |
def forward(self, x: Tensor) -> tuple[Tensor, Tensor, dict[str, Tensor]]:
|
| 154 |
features = self.backbone(x)
|
| 155 |
-
cls_logits, box_pred = self.head(features
|
| 156 |
return cls_logits, box_pred, features
|
| 157 |
|
| 158 |
|
|
@@ -202,27 +276,109 @@ def giou_loss(pred: Tensor, target: Tensor) -> Tensor:
|
|
| 202 |
return (1 - giou).mean()
|
| 203 |
|
| 204 |
|
| 205 |
-
def
|
| 206 |
-
|
| 207 |
-
|
| 208 |
-
|
| 209 |
-
|
| 210 |
-
|
| 211 |
-
|
| 212 |
-
|
| 213 |
-
|
| 214 |
-
|
| 215 |
-
|
| 216 |
-
|
| 217 |
-
|
| 218 |
-
|
| 219 |
-
|
| 220 |
-
|
| 221 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 222 |
else:
|
| 223 |
-
|
| 224 |
-
|
| 225 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 226 |
|
| 227 |
|
| 228 |
def nms(boxes: Tensor, scores: Tensor, iou_threshold: float, max_det: int) -> Tensor:
|
|
@@ -241,8 +397,8 @@ def nms(boxes: Tensor, scores: Tensor, iou_threshold: float, max_det: int) -> Te
|
|
| 241 |
|
| 242 |
|
| 243 |
def decode_predictions(cls_logits: Tensor, box_pred: Tensor, conf: float, topk: int, nms_iou: float, max_det: int) -> list[Tensor]:
|
| 244 |
-
scores = cls_logits.sigmoid()
|
| 245 |
-
boxes = box_pred
|
| 246 |
decoded = []
|
| 247 |
for score_row, box_row in zip(scores, boxes):
|
| 248 |
keep = score_row >= conf
|
|
@@ -356,7 +512,14 @@ def train() -> None:
|
|
| 356 |
val_loader = DataLoader(val_ds, batch_size=args.batch, shuffle=False, num_workers=args.workers, collate_fn=collate)
|
| 357 |
test_loader = DataLoader(test_ds, batch_size=args.batch, shuffle=False, num_workers=args.workers, collate_fn=collate)
|
| 358 |
|
| 359 |
-
model = DualBackboneDetector(
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 360 |
if args.data_parallel and torch.cuda.device_count() > 1:
|
| 361 |
model = nn.DataParallel(model)
|
| 362 |
raw = model.module if isinstance(model, nn.DataParallel) else model
|
|
@@ -372,30 +535,57 @@ def train() -> None:
|
|
| 372 |
best_recall = -1.0
|
| 373 |
history_path = output_dir / "history.csv"
|
| 374 |
with history_path.open("w", newline="", encoding="utf-8") as fp:
|
| 375 |
-
writer = csv.DictWriter(
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 376 |
writer.writeheader()
|
| 377 |
for epoch in range(1, args.epochs + 1):
|
| 378 |
raw.train()
|
| 379 |
losses = []
|
| 380 |
cls_losses = []
|
| 381 |
box_losses = []
|
|
|
|
|
|
|
| 382 |
for images, boxes_list, _ in train_loader:
|
| 383 |
images = images.to(device)
|
| 384 |
optimizer.zero_grad(set_to_none=True)
|
| 385 |
cls_logits, box_pred, _ = model(images)
|
| 386 |
-
loss, loss_items =
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 387 |
loss.backward()
|
| 388 |
torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
|
| 389 |
optimizer.step()
|
| 390 |
losses.append(float(loss.detach()))
|
| 391 |
cls_losses.append(loss_items["cls_loss"])
|
| 392 |
box_losses.append(loss_items["box_loss"])
|
|
|
|
|
|
|
| 393 |
val_metrics = evaluate(model, val_loader, device, args.conf, args.iou, args.topk, args.max_det)
|
| 394 |
row = {
|
| 395 |
"epoch": epoch,
|
| 396 |
"loss": sum(losses) / max(len(losses), 1),
|
| 397 |
"cls_loss": sum(cls_losses) / max(len(cls_losses), 1),
|
| 398 |
"box_loss": sum(box_losses) / max(len(box_losses), 1),
|
|
|
|
|
|
|
| 399 |
"val_precision": val_metrics.precision,
|
| 400 |
"val_recall": val_metrics.recall,
|
| 401 |
"val_mean_iou": val_metrics.mean_iou,
|
|
|
|
| 1 |
+
"""Train a DETR-style detector on top of the dual DINOv3 backbone."""
|
| 2 |
|
| 3 |
from __future__ import annotations
|
| 4 |
|
|
|
|
| 66 |
parser.add_argument("--iou", type=float, default=0.5)
|
| 67 |
parser.add_argument("--topk", type=int, default=300)
|
| 68 |
parser.add_argument("--max-det", type=int, default=100)
|
| 69 |
+
parser.add_argument("--num-queries", type=int, default=160)
|
| 70 |
+
parser.add_argument("--decoder-layers", type=int, default=4)
|
| 71 |
+
parser.add_argument("--decoder-heads", type=int, default=8)
|
| 72 |
+
parser.add_argument("--match-class-cost", type=float, default=2.0)
|
| 73 |
+
parser.add_argument("--match-l1-cost", type=float, default=5.0)
|
| 74 |
+
parser.add_argument("--match-giou-cost", type=float, default=2.0)
|
| 75 |
+
parser.add_argument("--no-object-weight", type=float, default=0.1)
|
| 76 |
parser.add_argument("--seed", type=int, default=0)
|
| 77 |
parser.add_argument("--no-pretrained", action="store_true")
|
| 78 |
parser.add_argument("--data-parallel", action="store_true")
|
|
|
|
| 128 |
return torch.stack(images), list(boxes), list(paths)
|
| 129 |
|
| 130 |
|
| 131 |
+
class MLP(nn.Module):
|
| 132 |
+
def __init__(self, input_dim: int, hidden_dim: int, output_dim: int, num_layers: int) -> None:
|
| 133 |
super().__init__()
|
| 134 |
+
layers = []
|
| 135 |
+
for idx in range(num_layers):
|
| 136 |
+
in_dim = input_dim if idx == 0 else hidden_dim
|
| 137 |
+
out_dim = output_dim if idx == num_layers - 1 else hidden_dim
|
| 138 |
+
layers.append(nn.Linear(in_dim, out_dim))
|
| 139 |
+
if idx < num_layers - 1:
|
| 140 |
+
layers.append(nn.GELU())
|
| 141 |
+
self.layers = nn.Sequential(*layers)
|
| 142 |
+
|
| 143 |
+
def forward(self, x: Tensor) -> Tensor:
|
| 144 |
+
return self.layers(x)
|
| 145 |
+
|
| 146 |
+
|
| 147 |
+
def sine_position_embedding(height: int, width: int, dim: int, device: torch.device, dtype: torch.dtype) -> Tensor:
|
| 148 |
+
if dim % 4 != 0:
|
| 149 |
+
raise ValueError("Position embedding dim must be divisible by 4")
|
| 150 |
+
y, x = torch.meshgrid(torch.arange(height, device=device), torch.arange(width, device=device), indexing="ij")
|
| 151 |
+
omega = torch.arange(dim // 4, device=device, dtype=dtype)
|
| 152 |
+
omega = 1.0 / (10000 ** (omega / max(dim // 4, 1)))
|
| 153 |
+
y = y.reshape(-1, 1).to(dtype) * omega
|
| 154 |
+
x = x.reshape(-1, 1).to(dtype) * omega
|
| 155 |
+
return torch.cat([torch.sin(y), torch.cos(y), torch.sin(x), torch.cos(x)], dim=1)
|
| 156 |
+
|
| 157 |
+
|
| 158 |
+
class SetPredictionHead(nn.Module):
|
| 159 |
+
"""Small DETR-style query decoder over fused semantic and detail memories."""
|
| 160 |
+
|
| 161 |
+
def __init__(
|
| 162 |
+
self,
|
| 163 |
+
in_channels: int = 256,
|
| 164 |
+
hidden: int = 256,
|
| 165 |
+
num_queries: int = 160,
|
| 166 |
+
decoder_layers: int = 4,
|
| 167 |
+
decoder_heads: int = 8,
|
| 168 |
+
) -> None:
|
| 169 |
+
super().__init__()
|
| 170 |
+
self.hidden = hidden
|
| 171 |
+
self.fused_proj = nn.Conv2d(in_channels, hidden, kernel_size=1)
|
| 172 |
+
self.detail_proj = nn.Conv2d(in_channels, hidden, kernel_size=1)
|
| 173 |
+
self.level_embed = nn.Parameter(torch.zeros(2, hidden))
|
| 174 |
+
self.query_embed = nn.Embedding(num_queries, hidden)
|
| 175 |
+
layer = nn.TransformerDecoderLayer(
|
| 176 |
+
d_model=hidden,
|
| 177 |
+
nhead=decoder_heads,
|
| 178 |
+
dim_feedforward=hidden * 4,
|
| 179 |
+
dropout=0.0,
|
| 180 |
+
activation="gelu",
|
| 181 |
+
batch_first=True,
|
| 182 |
+
norm_first=True,
|
| 183 |
)
|
| 184 |
+
self.decoder = nn.TransformerDecoder(layer, num_layers=decoder_layers)
|
| 185 |
+
self.class_embed = nn.Linear(hidden, 1)
|
| 186 |
+
self.box_embed = MLP(hidden, hidden, 4, num_layers=3)
|
| 187 |
+
nn.init.constant_(self.class_embed.bias, -2.0)
|
| 188 |
+
|
| 189 |
+
def _flatten_level(self, feat: Tensor, level: int) -> Tensor:
|
| 190 |
+
b, c, h, w = feat.shape
|
| 191 |
+
pos = sine_position_embedding(h, w, c, feat.device, feat.dtype)
|
| 192 |
+
tokens = feat.flatten(2).transpose(1, 2).contiguous()
|
| 193 |
+
return tokens + pos.unsqueeze(0) + self.level_embed[level].to(dtype=feat.dtype).view(1, 1, -1)
|
| 194 |
+
|
| 195 |
+
def forward(self, features: dict[str, Tensor]) -> tuple[Tensor, Tensor]:
|
| 196 |
+
fused = self.fused_proj(features["fused"])
|
| 197 |
+
detail = self.detail_proj(features["detail"])
|
| 198 |
+
memory = torch.cat([self._flatten_level(fused, 0), self._flatten_level(detail, 1)], dim=1)
|
| 199 |
+
queries = self.query_embed.weight.unsqueeze(0).expand(memory.shape[0], -1, -1)
|
| 200 |
+
decoded = self.decoder(tgt=queries, memory=memory)
|
| 201 |
+
return self.class_embed(decoded).squeeze(-1), self.box_embed(decoded).sigmoid()
|
| 202 |
|
| 203 |
|
| 204 |
class DualBackboneDetector(nn.Module):
|
| 205 |
+
def __init__(
|
| 206 |
+
self,
|
| 207 |
+
vit_weights: str,
|
| 208 |
+
convnext_weights: str,
|
| 209 |
+
pretrained: bool = True,
|
| 210 |
+
num_queries: int = 160,
|
| 211 |
+
decoder_layers: int = 4,
|
| 212 |
+
decoder_heads: int = 8,
|
| 213 |
+
) -> None:
|
| 214 |
super().__init__()
|
| 215 |
self.backbone = DINOv3ViTConvNeXtBackbone(
|
| 216 |
vit_weights=vit_weights,
|
| 217 |
convnext_weights=convnext_weights,
|
| 218 |
pretrained=pretrained,
|
| 219 |
)
|
| 220 |
+
self.head = SetPredictionHead(
|
| 221 |
+
in_channels=self.backbone.out_channels,
|
| 222 |
+
num_queries=num_queries,
|
| 223 |
+
decoder_layers=decoder_layers,
|
| 224 |
+
decoder_heads=decoder_heads,
|
| 225 |
+
)
|
| 226 |
|
| 227 |
def forward(self, x: Tensor) -> tuple[Tensor, Tensor, dict[str, Tensor]]:
|
| 228 |
features = self.backbone(x)
|
| 229 |
+
cls_logits, box_pred = self.head(features)
|
| 230 |
return cls_logits, box_pred, features
|
| 231 |
|
| 232 |
|
|
|
|
| 276 |
return (1 - giou).mean()
|
| 277 |
|
| 278 |
|
| 279 |
+
def linear_sum_assignment_torch(cost: Tensor) -> tuple[Tensor, Tensor]:
|
| 280 |
+
try:
|
| 281 |
+
from scipy.optimize import linear_sum_assignment
|
| 282 |
+
|
| 283 |
+
rows, cols = linear_sum_assignment(cost.detach().cpu().numpy())
|
| 284 |
+
return (
|
| 285 |
+
torch.as_tensor(rows, dtype=torch.long, device=cost.device),
|
| 286 |
+
torch.as_tensor(cols, dtype=torch.long, device=cost.device),
|
| 287 |
+
)
|
| 288 |
+
except Exception:
|
| 289 |
+
# Greedy fallback keeps training dependency-light. SciPy is preferred when available.
|
| 290 |
+
rows = []
|
| 291 |
+
cols = []
|
| 292 |
+
used_rows: set[int] = set()
|
| 293 |
+
used_cols: set[int] = set()
|
| 294 |
+
flat = cost.flatten().argsort()
|
| 295 |
+
n_cols = cost.shape[1]
|
| 296 |
+
for item in flat:
|
| 297 |
+
row = int(item // n_cols)
|
| 298 |
+
col = int(item % n_cols)
|
| 299 |
+
if row in used_rows or col in used_cols:
|
| 300 |
+
continue
|
| 301 |
+
used_rows.add(row)
|
| 302 |
+
used_cols.add(col)
|
| 303 |
+
rows.append(row)
|
| 304 |
+
cols.append(col)
|
| 305 |
+
if len(cols) == min(cost.shape):
|
| 306 |
+
break
|
| 307 |
+
return (
|
| 308 |
+
torch.as_tensor(rows, dtype=torch.long, device=cost.device),
|
| 309 |
+
torch.as_tensor(cols, dtype=torch.long, device=cost.device),
|
| 310 |
+
)
|
| 311 |
+
|
| 312 |
+
|
| 313 |
+
def generalized_iou_matrix(boxes1: Tensor, boxes2: Tensor) -> Tensor:
|
| 314 |
+
iou = box_iou_matrix(boxes1, boxes2)
|
| 315 |
+
if boxes1.numel() == 0 or boxes2.numel() == 0:
|
| 316 |
+
return iou
|
| 317 |
+
lt_c = torch.minimum(boxes1[:, None, :2], boxes2[None, :, :2])
|
| 318 |
+
rb_c = torch.maximum(boxes1[:, None, 2:], boxes2[None, :, 2:])
|
| 319 |
+
wh_c = (rb_c - lt_c).clamp(min=0)
|
| 320 |
+
area_c = (wh_c[..., 0] * wh_c[..., 1]).clamp(min=1e-8)
|
| 321 |
+
area1 = ((boxes1[:, 2] - boxes1[:, 0]).clamp(min=0) * (boxes1[:, 3] - boxes1[:, 1]).clamp(min=0))[:, None]
|
| 322 |
+
area2 = ((boxes2[:, 2] - boxes2[:, 0]).clamp(min=0) * (boxes2[:, 3] - boxes2[:, 1]).clamp(min=0))[None, :]
|
| 323 |
+
lt = torch.maximum(boxes1[:, None, :2], boxes2[None, :, :2])
|
| 324 |
+
rb = torch.minimum(boxes1[:, None, 2:], boxes2[None, :, 2:])
|
| 325 |
+
wh = (rb - lt).clamp(min=0)
|
| 326 |
+
inter = wh[..., 0] * wh[..., 1]
|
| 327 |
+
union = (area1 + area2 - inter).clamp(min=1e-8)
|
| 328 |
+
return iou - (area_c - union) / area_c
|
| 329 |
+
|
| 330 |
+
|
| 331 |
+
def set_prediction_loss(
|
| 332 |
+
pred_logits: Tensor,
|
| 333 |
+
pred_boxes: Tensor,
|
| 334 |
+
boxes_list: list[Tensor],
|
| 335 |
+
class_cost: float,
|
| 336 |
+
l1_cost: float,
|
| 337 |
+
giou_cost: float,
|
| 338 |
+
no_object_weight: float,
|
| 339 |
+
) -> tuple[Tensor, dict[str, float]]:
|
| 340 |
+
cls_targets = torch.zeros_like(pred_logits)
|
| 341 |
+
matched_pred_boxes = []
|
| 342 |
+
matched_target_boxes = []
|
| 343 |
+
total_targets = 0
|
| 344 |
+
|
| 345 |
+
for batch_idx, target_boxes_cpu in enumerate(boxes_list):
|
| 346 |
+
target_boxes = target_boxes_cpu.to(pred_boxes.device)
|
| 347 |
+
total_targets += int(target_boxes.shape[0])
|
| 348 |
+
if target_boxes.numel() == 0:
|
| 349 |
+
continue
|
| 350 |
+
prob = pred_logits[batch_idx].sigmoid()
|
| 351 |
+
class_match_cost = -prob[:, None].expand(-1, target_boxes.shape[0])
|
| 352 |
+
l1_match_cost = torch.cdist(pred_boxes[batch_idx], target_boxes, p=1)
|
| 353 |
+
giou_match_cost = -generalized_iou_matrix(pred_boxes[batch_idx], target_boxes)
|
| 354 |
+
cost = class_cost * class_match_cost + l1_cost * l1_match_cost + giou_cost * giou_match_cost
|
| 355 |
+
pred_idx, target_idx = linear_sum_assignment_torch(cost)
|
| 356 |
+
cls_targets[batch_idx, pred_idx] = 1.0
|
| 357 |
+
matched_pred_boxes.append(pred_boxes[batch_idx, pred_idx])
|
| 358 |
+
matched_target_boxes.append(target_boxes[target_idx])
|
| 359 |
+
|
| 360 |
+
cls_weight = torch.full_like(cls_targets, no_object_weight)
|
| 361 |
+
cls_weight = torch.where(cls_targets > 0, torch.ones_like(cls_weight), cls_weight)
|
| 362 |
+
cls_loss = F.binary_cross_entropy_with_logits(pred_logits, cls_targets, weight=cls_weight, reduction="sum")
|
| 363 |
+
cls_loss = cls_loss / max(total_targets, 1)
|
| 364 |
+
|
| 365 |
+
if matched_pred_boxes:
|
| 366 |
+
pred_cat = torch.cat(matched_pred_boxes, dim=0)
|
| 367 |
+
target_cat = torch.cat(matched_target_boxes, dim=0)
|
| 368 |
+
l1_loss = F.l1_loss(pred_cat, target_cat, reduction="none").sum() / max(total_targets, 1)
|
| 369 |
+
giou = generalized_iou_matrix(pred_cat, target_cat).diag()
|
| 370 |
+
giou_loss_value = (1.0 - giou).sum() / max(total_targets, 1)
|
| 371 |
else:
|
| 372 |
+
l1_loss = pred_boxes.sum() * 0
|
| 373 |
+
giou_loss_value = pred_boxes.sum() * 0
|
| 374 |
+
|
| 375 |
+
total = cls_loss + l1_cost * l1_loss + giou_cost * giou_loss_value
|
| 376 |
+
return total, {
|
| 377 |
+
"cls_loss": float(cls_loss.detach()),
|
| 378 |
+
"box_loss": float((l1_loss + giou_loss_value).detach()),
|
| 379 |
+
"l1_loss": float(l1_loss.detach()),
|
| 380 |
+
"giou_loss": float(giou_loss_value.detach()),
|
| 381 |
+
}
|
| 382 |
|
| 383 |
|
| 384 |
def nms(boxes: Tensor, scores: Tensor, iou_threshold: float, max_det: int) -> Tensor:
|
|
|
|
| 397 |
|
| 398 |
|
| 399 |
def decode_predictions(cls_logits: Tensor, box_pred: Tensor, conf: float, topk: int, nms_iou: float, max_det: int) -> list[Tensor]:
|
| 400 |
+
scores = cls_logits.sigmoid()
|
| 401 |
+
boxes = box_pred
|
| 402 |
decoded = []
|
| 403 |
for score_row, box_row in zip(scores, boxes):
|
| 404 |
keep = score_row >= conf
|
|
|
|
| 512 |
val_loader = DataLoader(val_ds, batch_size=args.batch, shuffle=False, num_workers=args.workers, collate_fn=collate)
|
| 513 |
test_loader = DataLoader(test_ds, batch_size=args.batch, shuffle=False, num_workers=args.workers, collate_fn=collate)
|
| 514 |
|
| 515 |
+
model = DualBackboneDetector(
|
| 516 |
+
args.vit_weights,
|
| 517 |
+
args.convnext_weights,
|
| 518 |
+
pretrained=not args.no_pretrained,
|
| 519 |
+
num_queries=args.num_queries,
|
| 520 |
+
decoder_layers=args.decoder_layers,
|
| 521 |
+
decoder_heads=args.decoder_heads,
|
| 522 |
+
).to(device)
|
| 523 |
if args.data_parallel and torch.cuda.device_count() > 1:
|
| 524 |
model = nn.DataParallel(model)
|
| 525 |
raw = model.module if isinstance(model, nn.DataParallel) else model
|
|
|
|
| 535 |
best_recall = -1.0
|
| 536 |
history_path = output_dir / "history.csv"
|
| 537 |
with history_path.open("w", newline="", encoding="utf-8") as fp:
|
| 538 |
+
writer = csv.DictWriter(
|
| 539 |
+
fp,
|
| 540 |
+
fieldnames=[
|
| 541 |
+
"epoch",
|
| 542 |
+
"loss",
|
| 543 |
+
"cls_loss",
|
| 544 |
+
"box_loss",
|
| 545 |
+
"l1_loss",
|
| 546 |
+
"giou_loss",
|
| 547 |
+
"val_precision",
|
| 548 |
+
"val_recall",
|
| 549 |
+
"val_mean_iou",
|
| 550 |
+
],
|
| 551 |
+
)
|
| 552 |
writer.writeheader()
|
| 553 |
for epoch in range(1, args.epochs + 1):
|
| 554 |
raw.train()
|
| 555 |
losses = []
|
| 556 |
cls_losses = []
|
| 557 |
box_losses = []
|
| 558 |
+
l1_losses = []
|
| 559 |
+
giou_losses = []
|
| 560 |
for images, boxes_list, _ in train_loader:
|
| 561 |
images = images.to(device)
|
| 562 |
optimizer.zero_grad(set_to_none=True)
|
| 563 |
cls_logits, box_pred, _ = model(images)
|
| 564 |
+
loss, loss_items = set_prediction_loss(
|
| 565 |
+
cls_logits,
|
| 566 |
+
box_pred,
|
| 567 |
+
boxes_list,
|
| 568 |
+
class_cost=args.match_class_cost,
|
| 569 |
+
l1_cost=args.match_l1_cost,
|
| 570 |
+
giou_cost=args.match_giou_cost,
|
| 571 |
+
no_object_weight=args.no_object_weight,
|
| 572 |
+
)
|
| 573 |
loss.backward()
|
| 574 |
torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
|
| 575 |
optimizer.step()
|
| 576 |
losses.append(float(loss.detach()))
|
| 577 |
cls_losses.append(loss_items["cls_loss"])
|
| 578 |
box_losses.append(loss_items["box_loss"])
|
| 579 |
+
l1_losses.append(loss_items["l1_loss"])
|
| 580 |
+
giou_losses.append(loss_items["giou_loss"])
|
| 581 |
val_metrics = evaluate(model, val_loader, device, args.conf, args.iou, args.topk, args.max_det)
|
| 582 |
row = {
|
| 583 |
"epoch": epoch,
|
| 584 |
"loss": sum(losses) / max(len(losses), 1),
|
| 585 |
"cls_loss": sum(cls_losses) / max(len(cls_losses), 1),
|
| 586 |
"box_loss": sum(box_losses) / max(len(box_losses), 1),
|
| 587 |
+
"l1_loss": sum(l1_losses) / max(len(l1_losses), 1),
|
| 588 |
+
"giou_loss": sum(giou_losses) / max(len(giou_losses), 1),
|
| 589 |
"val_precision": val_metrics.precision,
|
| 590 |
"val_recall": val_metrics.recall,
|
| 591 |
"val_mean_iou": val_metrics.mean_iou,
|