File size: 26,642 Bytes
519ab98 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 | """
论文 §4.3 段级动态多尺度融合 — PyTorch 可训练封装。
§4.3.1 角向下采样 → 共享 Φ_ms + FiLM(k) → 段内对齐 H̃_k^(m)
§4.3.2 频率引导尺度注意力(Query: f̄_freq, g_vec, E_pen)
§4.3.3 段融合 → Transformer → f_ellipse
默认 K={1,2,4,8,16,32},D_h=128,L_ell=3(与 §7.1.3 一致)。
"""
from __future__ import annotations
from dataclasses import dataclass, field
from pathlib import Path
from typing import Dict, List, Optional, Sequence, Tuple, Union
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
from elipse_feature import DEFAULT_IMG_SIZE, FeaTensor
from section41 import (
Section41Config,
Section41Encoder,
build_u_man,
extract_section41,
fea_fields_from_dataloader,
img_tensor_to_bgr,
keypoints_px_from_u_man,
)
from section42 import (
Section42Config,
Section42Encoder,
build_segment_geometry,
responsibility_indices,
)
PathLike = Union[str, Path]
DEFAULT_SCALES: Tuple[int, ...] = (1, 2, 4, 8, 16, 32)
@dataclass
class Section43Config:
n_theta: int = 360
n_segments: int = 10
scales: Tuple[int, ...] = DEFAULT_SCALES
d_seq: int = 128
c1: int = 64
d_h: int = 128
d_e: int = 16
d_freq: int = 64
d_geo: int = 32
d_pen: int = 16
h_scale: int = 4
d_prime: int = 32
l_ell: int = 3
n_heads_ell: int = 4
ffn_ell: int = 256
conv1_kernel: int = 7
# ---------------------------------------------------------------------------
# §4.3.1.1 角向下采样
# ---------------------------------------------------------------------------
def compute_Lk_raw(n_theta: int, k: int) -> int:
"""
角向下采样后粗序列长度 L_k^raw ≈ N_θ / k。
推理:角标 t_i = i·k,i = 0, …, L-1,L = ⌊N_θ/k⌋(N_θ=360 时 k∈{8,16,32} 恰为 45/22/11)。
旧实现用训练约束公式 I_k=⌊(N-1)/k⌋-(k-1),在 k≥8 时严重偏短,已弃用。
"""
return max(1, n_theta // k)
def multiscale_sample_indices(
n_theta: int,
k: int,
*,
training: bool = False,
eps: Optional[int] = None,
) -> torch.Tensor:
"""
返回角标序列 (L_k^raw,),长度 L ≈ N_θ/k。
推理:t_i = i·k,i=0,…,L-1。
训练:t_i = (i+ε)·k,ε∼U{0,…,k-1} 与同次前向共享,再 clip 到 [0, N_θ-1]。
"""
L = compute_Lk_raw(n_theta, k)
if training:
e = int(torch.randint(0, k, (1,)).item()) if eps is None else int(eps)
idx = [min((i + e) * k, n_theta - 1) for i in range(L)]
else:
idx = [i * k for i in range(L)]
return torch.tensor(idx, dtype=torch.long)
# ---------------------------------------------------------------------------
# §4.3.1.2 共享多尺度 CNN + FiLM
# ---------------------------------------------------------------------------
class SharedMultiscaleStem(nn.Module):
"""ψ^(1) + FiLM(k) + MaxPool + ψ^tail,输入 (B, L, d_seq) → (B, L_pool, d_h)。"""
def __init__(self, cfg: Section43Config, n_scales: int):
super().__init__()
self.cfg = cfg
pad1 = cfg.conv1_kernel // 2
self.conv1 = nn.Conv1d(cfg.d_seq, cfg.c1, kernel_size=cfg.conv1_kernel, padding=pad1)
self.scale_embed = nn.Embedding(n_scales, cfg.d_e)
self.film_gamma = nn.Linear(cfg.d_e, cfg.c1)
self.film_beta = nn.Linear(cfg.d_e, cfg.c1)
self.pool = nn.MaxPool1d(kernel_size=2, stride=2)
pad3 = 1
self.conv2 = nn.Conv1d(cfg.c1, cfg.d_h, kernel_size=3, padding=pad3)
self.conv3 = nn.Conv1d(cfg.d_h, cfg.d_h, kernel_size=3, padding=pad3)
self.act = nn.GELU()
def forward(self, bar_s: torch.Tensor, scale_id: int) -> torch.Tensor:
"""
bar_s: (B, L_raw, d_seq)
返回 H_k: (B, L_pool, d_h)
"""
x = bar_s.transpose(1, 2)
z = self.act(self.conv1(x))
e = self.scale_embed(torch.tensor(scale_id, device=z.device, dtype=torch.long))
gamma = self.film_gamma(e).unsqueeze(-1)
beta = self.film_beta(e).unsqueeze(-1)
z = gamma * z + beta
z = self.pool(z)
z = self.act(self.conv2(z))
z = self.act(self.conv3(z))
return z.transpose(1, 2)
def _interp1d_positions(feat: torch.Tensor, positions: torch.Tensor) -> torch.Tensor:
"""
feat: (B, L, C),positions: (s_l,) 浮点索引 ∈ [0, L-1]
返回 (B, s_l, C)
"""
b, L, c = feat.shape
s_l = positions.shape[0]
pos = positions.clamp(0, max(L - 1, 0))
i0 = pos.long()
i1 = torch.clamp(i0 + 1, max=L - 1)
w = (pos - i0.float()).view(1, s_l, 1)
f0 = feat[:, i0, :]
f1 = feat[:, i1, :]
return f0 * (1.0 - w) + f1 * w
def align_H_to_segment(
H_k: torch.Tensor,
seg_theta_idx: np.ndarray,
n_theta: int,
k: int,
) -> torch.Tensor:
"""
§4.3.1.3:H_k (B, L_pool, d_h) 对齐到段内 s_l 个角锚 → (B, s_l, d_h)。
角锚 θ 映射到粗网格池化后坐标:θ / k / 2。
"""
L_pool = H_k.shape[1]
pos = torch.tensor(seg_theta_idx, dtype=torch.float32, device=H_k.device) / float(k) / 2.0
pos = pos.clamp(0, max(L_pool - 1, 0))
return _interp1d_positions(H_k, pos)
# ---------------------------------------------------------------------------
# §4.3.2 频率引导尺度注意力
# ---------------------------------------------------------------------------
class FreqGuidedScaleAttention(nn.Module):
def __init__(self, cfg: Section43Config, n_scales: int):
super().__init__()
self.cfg = cfg
self.n_scales = n_scales
d_qinp = cfg.d_freq + cfg.d_geo + cfg.d_pen
self.q_proj = nn.Linear(d_qinp, cfg.h_scale * cfg.d_prime)
self.w_k = nn.ModuleList(
[nn.Linear(cfg.d_h, cfg.d_prime, bias=False) for _ in range(cfg.h_scale)]
)
def forward(
self,
f_bar: torch.Tensor,
g_vec: torch.Tensor,
e_pen: torch.Tensor,
t_stack: torch.Tensor,
) -> Tuple[torch.Tensor, torch.Tensor]:
"""
f_bar: (B, d_freq) 段级频域均值
g_vec: (B, d_geo)
e_pen: (B, d_pen)
t_stack: (B, n_scales, d_h) 各尺度段摘要 t_k^(m)
返回:
W_scale: (B, n_scales) 归一化尺度权重
beta_heads: (B, h_scale, n_scales)
"""
u = torch.cat([f_bar, g_vec, e_pen], dim=-1)
q_all = self.q_proj(u)
q_heads = q_all.view(-1, self.cfg.h_scale, self.cfg.d_prime)
beta_list = []
scale = self.cfg.d_prime**0.5
for hi in range(self.cfg.h_scale):
q = q_heads[:, hi, :]
k_proj = self.w_k[hi](t_stack)
logits = torch.einsum("bd,bkd->bk", q, k_proj) / scale
beta_list.append(F.softmax(logits, dim=-1))
beta_heads = torch.stack(beta_list, dim=1)
beta_mean = beta_heads.mean(dim=1)
W_scale = beta_mean / (beta_mean.sum(dim=-1, keepdim=True) + 1e-12)
return W_scale, beta_heads
# ---------------------------------------------------------------------------
# §4.3 完整编码器
# ---------------------------------------------------------------------------
class Section43Encoder(nn.Module):
"""§4.3 段级动态多尺度融合(全部可训练)。"""
def __init__(self, config: Optional[Section43Config] = None):
super().__init__()
self.config = config or Section43Config()
cfg = self.config
self.scales = list(cfg.scales)
self.n_scales = len(self.scales)
self.geom = build_segment_geometry(cfg.n_theta, cfg.n_segments)
self.s_l = self.geom.s_l
self.M = self.geom.n_segments
self.ms_stem = SharedMultiscaleStem(cfg, self.n_scales)
self.scale_attn = FreqGuidedScaleAttention(cfg, self.n_scales)
enc_layer = nn.TransformerEncoderLayer(
d_model=cfg.d_h,
nhead=cfg.n_heads_ell,
dim_feedforward=cfg.ffn_ell,
batch_first=True,
norm_first=True,
activation="gelu",
)
self.ell_transformer = nn.TransformerEncoder(enc_layer, num_layers=cfg.l_ell)
def _encode_scale(
self,
s_seq: torch.Tensor,
k: int,
scale_id: int,
training: bool,
) -> torch.Tensor:
"""单尺度 H_k: (B, L_pool, d_h)。"""
b, n, _ = s_seq.shape
idx = multiscale_sample_indices(n, k, training=training).to(s_seq.device)
bar_s = s_seq[:, idx, :]
return self.ms_stem(bar_s, scale_id)
def _align_all_segments(
self,
H_k: torch.Tensor,
k: int,
) -> torch.Tensor:
"""(B, M, s_l, d_h)。"""
b = H_k.shape[0]
out = []
for m in range(self.M):
seg_idx = responsibility_indices(m, self.geom)
h_m = align_H_to_segment(H_k, seg_idx, self.config.n_theta, k)
out.append(h_m.unsqueeze(1))
return torch.cat(out, dim=1)
def forward(
self,
s_seq: torch.Tensor,
f_freq: torch.Tensor,
g_vec: torch.Tensor,
e_pen: Optional[torch.Tensor] = None,
*,
training: bool = False,
return_details: bool = True,
) -> Dict[str, torch.Tensor]:
"""
s_seq: (B, N_theta, d_seq)
f_freq: (B, M, d_freq, s_l) — 与 section42 输出一致
g_vec: (B, d_geo)
e_pen: (B, d_pen) 或 None(未知笔型用零向量)
"""
cfg = self.config
batched = s_seq.dim() == 3
if not batched:
s_seq = s_seq.unsqueeze(0)
f_freq = f_freq.unsqueeze(0)
g_vec = g_vec.unsqueeze(0)
b = s_seq.size(0)
device = s_seq.device
if e_pen is None:
e_pen = torch.zeros(b, cfg.d_pen, device=device, dtype=s_seq.dtype)
elif e_pen.dim() == 1:
e_pen = e_pen.unsqueeze(0).expand(b, -1)
f_bar = f_freq.mean(dim=-1)
H_aligned: List[torch.Tensor] = []
for sid, k in enumerate(self.scales):
H_k = self._encode_scale(s_seq, k, sid, training)
H_tilde = self._align_all_segments(H_k, k)
H_aligned.append(H_tilde)
H_stack = torch.stack(H_aligned, dim=2)
F_seg = torch.zeros(b, self.M, self.s_l, cfg.d_h, device=device, dtype=s_seq.dtype)
W_all = torch.zeros(b, self.M, self.n_scales, device=device, dtype=s_seq.dtype)
beta_all = torch.zeros(b, self.M, cfg.h_scale, self.n_scales, device=device)
for m in range(self.M):
t_stack = H_stack[:, m, :, :].mean(dim=2)
W_m, beta_m = self.scale_attn(f_bar[:, m], g_vec, e_pen, t_stack)
W_all[:, m] = W_m
beta_all[:, m] = beta_m
for sid in range(self.n_scales):
F_seg[:, m] = F_seg[:, m] + W_m[:, sid : sid + 1, None] * H_stack[:, m, sid]
H_ell = F_seg.reshape(b, self.M * self.s_l, cfg.d_h)
H_ell = self.ell_transformer(H_ell)
f_ellipse = H_ell.max(dim=1).values
out: Dict[str, torch.Tensor] = {
"f_ellipse": f_ellipse,
"H_ell": H_ell,
"F_segment": F_seg,
"W_scale": W_all,
}
if return_details:
out["H_aligned"] = H_stack
out["beta_heads"] = beta_all
out["f_bar"] = f_bar
if not batched:
out = {k: v.squeeze(0) for k, v in out.items()}
return out
def forward_from_upstream(
self,
o41: Dict[str, torch.Tensor],
o42: Dict[str, torch.Tensor],
e_pen: Optional[torch.Tensor] = None,
*,
training: bool = False,
return_details: bool = True,
**kwargs,
) -> Dict[str, torch.Tensor]:
"""§4.1 + §4.2 输出字典 → §4.3(s_seq / f_freq / g_vec)。"""
return self.forward(
o41["s_seq"],
o42["f_freq"],
o41["g_vec"],
e_pen,
training=training,
return_details=return_details,
)
def save_checkpoint(self, path: Union[str, Path], extra: Optional[dict] = None) -> None:
torch.save(
{"config": self.config, "state_dict": self.state_dict(), "extra": extra or {}},
path,
)
@classmethod
def load_checkpoint(
cls,
path: Union[str, Path],
map_location: Optional[str] = None,
strict: bool = True,
) -> "Section43Encoder":
payload = torch.load(path, map_location=map_location, weights_only=False)
cfg = payload.get("config") or Section43Config()
if isinstance(cfg, dict):
cfg = Section43Config(**cfg)
model = cls(cfg)
model.load_state_dict(payload["state_dict"], strict=strict)
return model
class EllipseBranch43(nn.Module):
"""§4.1 → §4.2 → §4.3 串联(训练时可仅优化 section43 或端到端)。"""
def __init__(
self,
cfg41: Optional[Section41Config] = None,
cfg42: Optional[Section42Config] = None,
cfg43: Optional[Section43Config] = None,
*,
freeze41_deep: bool = True,
):
super().__init__()
self.enc41 = Section41Encoder(cfg41)
self.enc42 = Section42Encoder(cfg42)
self.enc43 = Section43Encoder(cfg43)
if freeze41_deep:
for p in self.enc41.deep_branch.parameters():
p.requires_grad = False
def forward(
self,
rgb: PathLike,
e_pen: Optional[torch.Tensor] = None,
*,
training: bool = False,
device: Optional[torch.device] = None,
) -> Dict[str, torch.Tensor]:
from elipse_feature import extract_ellipse_features, preprocess_rgb_to_skeleton
prep = preprocess_rgb_to_skeleton(rgb, size=self.enc41.config.img_size)
bundle = extract_ellipse_features(
prep["skeleton"],
n_theta=self.enc43.config.n_theta,
half_norm=self.enc41.config.img_size / 2.0,
)
s_res = torch.from_numpy(bundle.sequence.s_res).float()
if device:
s_res = s_res.to(device)
with torch.no_grad():
h_deep = self.enc41.deep_branch(
prep["rgb"],
bundle.sequence.s_base_px,
patch_size=self.enc41.config.patch_size,
)
g = torch.from_numpy(bundle.global_feat.as_vector()).float()
u = torch.from_numpy(build_u_man(bundle.sequence)).float()
if device:
g, u, h_deep = g.to(device), u.to(device), h_deep.to(device)
o41 = self.enc41.forward_tensors(g, u, h_deep)
o42 = self.enc42(s_res.unsqueeze(0) if s_res.dim() == 2 else s_res)
s_seq = o41["s_seq"]
if s_seq.dim() == 2:
s_seq = s_seq.unsqueeze(0)
g_vec = o41["g_vec"].unsqueeze(0)
f_freq = o42["f_freq"].unsqueeze(0) if o42["f_freq"].dim() == 3 else o42["f_freq"]
else:
g_vec = o41["g_vec"]
f_freq = o42["f_freq"]
o43 = self.enc43(
s_seq,
f_freq,
g_vec,
e_pen,
training=training,
)
return {"prep": prep, "bundle": bundle, **{f"o41_{k}": v for k, v in o41.items()}, **{f"o42_{k}": v for k, v in o42.items()}, **o43}
def extract_section43(
s_seq: torch.Tensor,
f_freq: torch.Tensor,
g_vec: torch.Tensor,
e_pen: Optional[torch.Tensor] = None,
config: Optional[Section43Config] = None,
encoder: Optional[Section43Encoder] = None,
device: Optional[str] = None,
training: bool = False,
) -> Dict[str, torch.Tensor]:
enc = encoder or Section43Encoder(config)
if device:
enc = enc.to(device)
s_seq = s_seq.to(device)
f_freq = f_freq.to(device)
g_vec = g_vec.to(device)
enc.eval() if not training else enc.train()
return enc(s_seq, f_freq, g_vec, e_pen, training=training)
def forward_pipeline_from_dataloader(
fea_tensor: FeaTensor,
img_tensor: torch.Tensor,
enc41: Section41Encoder,
enc42: Section42Encoder,
enc43: Section43Encoder,
*,
half_norm: float = DEFAULT_IMG_SIZE / 2.0,
e_pen: Optional[torch.Tensor] = None,
training: bool = False,
) -> Tuple[Dict[str, torch.Tensor], Dict[str, torch.Tensor], Dict[str, torch.Tensor]]:
"""
DataLoader 单样本:fea_tensor + img_tensor → §4.1 → §4.2 → §4.3。
"""
fields = fea_fields_from_dataloader(fea_tensor)
if fields.u_man.dim() == 3:
raise NotImplementedError("单样本 FeaTensor 请传入 (N,8) 而非 batch 维")
device = next(enc43.parameters()).device
g_t = fields.g_global.float().to(device)
u_t = fields.u_man.float().to(device)
pts = keypoints_px_from_u_man(u_t, half_norm=half_norm)
bgr = img_tensor_to_bgr(img_tensor)
h_deep = enc41.extract_h_deep(bgr, pts)
o41 = enc41.forward_tensors(g_t, u_t, h_deep)
o42 = enc42.forward_from_fea(fea_tensor)
enc43_kwargs = build_enc43_valid_mask_kwargs(enc43, fea_tensor, device)
o43 = enc43.forward_from_upstream(
o41, o42, e_pen, training=training, return_details=not training, **enc43_kwargs
)
return o41, o42, o43
def build_enc43_valid_mask_kwargs(
enc43: Section43Encoder,
fea_tensor: FeaTensor,
device: torch.device,
) -> Dict[str, torch.Tensor]:
"""为 enc43.forward_from_upstream 构造 valid_mask(v2/v3.1)。"""
enc43_kwargs: Dict[str, torch.Tensor] = {}
try:
from section43_v2 import build_pool_valid_mask, v2_needs_valid_mask
from section43_v3_1 import v3_1_needs_valid_mask
if v2_needs_valid_mask(enc43) or v3_1_needs_valid_mask(enc43):
valid = fea_tensor.fields.valid.float().to(device)
if valid.dim() == 1:
valid = valid.unsqueeze(0)
if getattr(enc43.config, "stage_pool_mode", None) in ("cwt_angular", "cwt_dual"):
enc43_kwargs["valid_mask"] = valid
else:
enc43_kwargs["valid_mask"] = build_pool_valid_mask(
valid,
enc43.geom,
enc43.M,
enc43.s_l,
)
except ImportError:
pass
return enc43_kwargs
def forward_pipeline_batch_to_upstream(
fea_tensor: FeaTensor,
img_tensor: torch.Tensor,
enc41: Section41Encoder,
enc42: Section42Encoder,
*,
half_norm: float = DEFAULT_IMG_SIZE / 2.0,
) -> Tuple[Dict[str, torch.Tensor], Dict[str, torch.Tensor]]:
"""DataLoader batch:§4.1 + §4.2,不含 §4.3(供 CWT 等扩展在 enc43 前注入 o42)。"""
device = next(enc42.parameters()).device
fields = fea_tensor.fields
g_b = fields.g_global.float().to(device)
u_b = fields.u_man.float().to(device)
if enc41.deep_branch.backend == "resnet":
h_b = enc41.extract_h_deep_batch(img_tensor, u_b, half_norm=half_norm)
else:
h_list = []
for i in range(img_tensor.shape[0]):
pts = keypoints_px_from_u_man(u_b[i], half_norm=half_norm)
bgr = img_tensor_to_bgr(img_tensor[i])
h_list.append(enc41.extract_h_deep(bgr, pts))
h_b = torch.stack(h_list, dim=0)
o41 = enc41.forward_tensors(g_b, u_b, h_b)
s_res_b = fields.s_res.float().to(device)
o42 = enc42.forward(
s_res_b,
amp_segment=fields.amp_segment.float().to(device),
amp_global=fields.amp_global.float().to(device),
)
return o41, o42
def forward_pipeline_batch_from_dataloader(
fea_tensor: FeaTensor,
img_tensor: torch.Tensor,
enc41: Section41Encoder,
enc42: Section42Encoder,
enc43: Section43Encoder,
*,
half_norm: float = DEFAULT_IMG_SIZE / 2.0,
e_pen: Optional[torch.Tensor] = None,
training: bool = False,
) -> Tuple[Dict[str, torch.Tensor], Dict[str, torch.Tensor], Dict[str, torch.Tensor]]:
"""DataLoader batch:逐样本 deep + batched §4.1/§4.2/§4.3(§4.3 ConvNeXt1D 路径未改)。"""
device = next(enc43.parameters()).device
o41, o42 = forward_pipeline_batch_to_upstream(
fea_tensor, img_tensor, enc41, enc42, half_norm=half_norm
)
enc43_kwargs = build_enc43_valid_mask_kwargs(enc43, fea_tensor, device)
o43 = enc43.forward_from_upstream(
o41, o42, e_pen, training=training, return_details=not training, **enc43_kwargs
)
return o41, o42, o43
def demo_from_dataloader(
sample_ratio: float = 0.001,
batch_size: int = 2,
half_norm: float = DEFAULT_IMG_SIZE / 2.0,
cfg41: Optional[Section41Config] = None,
cfg42: Optional[Section42Config] = None,
cfg43: Optional[Section43Config] = None,
device: str = "cpu",
deep_backend: str = "resnet",
) -> Dict[str, torch.Tensor]:
"""
从 exp6_data_loader 取样本:§4.1(s_seq, g_vec)+ §4.2(f_freq)→ §4.3。
"""
from exp6_data_loader import create_dataloaders
cfg41 = cfg41 or Section41Config(
deep_backend=deep_backend,
resnet_checkpoint=(
None
if deep_backend == "patch"
else Section41Config().resnet_checkpoint
),
)
cfg42 = cfg42 or Section42Config()
cfg43 = cfg43 or Section43Config()
enc41 = Section41Encoder(cfg41).to(device)
enc42 = Section42Encoder(cfg42).to(device)
enc43 = Section43Encoder(cfg43).to(device)
enc41.eval()
enc42.eval()
enc43.eval()
train_loader, _, _, _ = create_dataloaders(
batch_size=batch_size,
num_workers=0,
sample_ratio=sample_ratio,
verbose=False,
)
batch = next(iter(train_loader))
input_all, labels, pen_ids, writers, image_ids = batch
fea_tensor, ske_tensor, img_tensor = input_all
print("[section43] dataloader batch")
print(f" fea_tensor.data: {fea_tensor.data.shape}")
print(f" img_tensor: {tuple(img_tensor.shape)} label: {labels.tolist()}")
print(f" deep_backend: {cfg41.deep_backend} scales: {cfg43.scales}")
fea_one = FeaTensor(data=fea_tensor.data[0])
img_one = img_tensor[0]
fields = fea_one.fields
print(f" g_global: {tuple(fields.g_global.shape)} u_man: {tuple(fields.u_man.shape)}")
print(f" amp_segment: {tuple(fields.amp_segment.shape)} amp_global: {tuple(fields.amp_global.shape)}")
with torch.no_grad():
o41, o42, o43 = forward_pipeline_from_dataloader(
fea_one,
img_one,
enc41,
enc42,
enc43,
half_norm=half_norm,
)
n41 = sum(p.numel() for p in enc41.trainable_parameters())
n42 = sum(p.numel() for p in enc42.trainable_parameters())
n43 = sum(p.numel() for p in enc43.parameters())
print("[section43] pipeline single (41→42→43)")
print(f" writer: {writers[0]} image_id: {image_ids[0]}")
print(f" trainable §4.1: {n41:,} §4.2: {n42:,} §4.3: {n43:,}")
print(f" s_seq: {tuple(o41['s_seq'].shape)}")
print(f" g_vec: {tuple(o41['g_vec'].shape)}")
print(f" f_freq: {tuple(o42['f_freq'].shape)}")
print(f" f_bar: {tuple(o42['f_bar'].shape)}")
print(f" f_ellipse: {tuple(o43['f_ellipse'].shape)}")
print(f" F_segment: {tuple(o43['F_segment'].shape)}")
print(f" W_scale: {tuple(o43['W_scale'].shape)}")
print(f" H_ell: {tuple(o43['H_ell'].shape)}")
ws = o43["W_scale"]
if ws.dim() == 2:
w0 = ws[0].detach().cpu().numpy()
else:
w0 = ws.detach().cpu().numpy().ravel()
print(f" 段0尺度权重: {dict(zip(DEFAULT_SCALES, [f'{float(x):.3f}' for x in w0]))}")
print("[section43] pipeline batch")
with torch.no_grad():
o41_b, o42_b, o43_b = forward_pipeline_batch_from_dataloader(
fea_tensor,
img_tensor,
enc41,
enc42,
enc43,
half_norm=half_norm,
)
print(f" batch s_seq: {tuple(o41_b['s_seq'].shape)}")
print(f" batch f_freq: {tuple(o42_b['f_freq'].shape)}")
print(f" batch f_ellipse: {tuple(o43_b['f_ellipse'].shape)}")
print(f" batch W_scale: {tuple(o43_b['W_scale'].shape)}")
return o43
def demo(image_path: str = "data/images/20938.png", device: str = "cpu") -> Dict[str, torch.Tensor]:
branch = EllipseBranch43().to(device)
branch.eval()
with torch.no_grad():
out = branch.forward(image_path, device=torch.device(device))
n_params = sum(p.numel() for p in branch.enc43.parameters())
print(f"§4.3 demo: {image_path}")
print(f" enc43 参数: {n_params:,}")
print(f" f_ellipse {out['f_ellipse'].shape}")
print(f" W_scale {out['W_scale'].shape}")
print(f" F_segment {out['F_segment'].shape}")
print(f" H_ell {out['H_ell'].shape}")
ws = out["W_scale"]
if ws.dim() == 3:
ws = ws[0, 0]
elif ws.dim() == 2:
ws = ws[0]
w0 = ws.detach().cpu().numpy().ravel()
print(f" 段0尺度权重: {dict(zip(DEFAULT_SCALES, [f'{float(x):.3f}' for x in w0]))}")
return out
def main() -> None:
import argparse
p = argparse.ArgumentParser(description="§4.3 multiscale fusion encoder")
p.add_argument("image", nargs="?", default="data/images/20938.png")
p.add_argument("--device", default="cpu")
p.add_argument("--save", type=str, default=None)
p.add_argument("--ckpt", type=str, default=None)
p.add_argument(
"--from-dataloader",
action="store_true",
help="使用 exp6_data_loader → §4.1 → §4.2 → §4.3 调试",
)
p.add_argument("--sample-ratio", type=float, default=0.001)
p.add_argument("--batch-size", type=int, default=2)
p.add_argument(
"--deep",
choices=("resnet", "patch"),
default="resnet",
help=f"§4.1.3 深度分支(默认 resnet + 平台默认权重)",
)
args = p.parse_args()
if args.from_dataloader:
demo_from_dataloader(
sample_ratio=args.sample_ratio,
batch_size=args.batch_size,
device=args.device,
deep_backend=args.deep,
)
return
out = demo(args.image, args.device)
if args.ckpt:
EllipseBranch43().enc43.save_checkpoint(args.ckpt)
print("saved enc43:", args.ckpt)
if args.save:
np.savez(
args.save,
f_ellipse=out["f_ellipse"].cpu().numpy(),
W_scale=out["W_scale"].cpu().numpy(),
)
print("saved:", args.save)
if __name__ == "__main__":
demo_from_dataloader()
|