Download code/section41.py from qinxunhui/deepellipse-circleid: direct link, hf CLI and curl.
- Browser
- Download file 27.7 kB
-
https://huggingface.co/qinxunhui/deepellipse-circleid/resolve/main/code/section41.py
- Command line
-
hf download hf://qinxunhui/deepellipse-circleid/code/section41.py
-
curl -L -o section41.py https://huggingface.co/qinxunhui/deepellipse-circleid/resolve/main/code/section41.py
27.7 kB
| """ | |
| 论文 §4.1 椭圆几何建模与序列化表征 — PyTorch 可训练封装。 | |
| §4.1.1 MLP_geo(g_global) → g_vec [可训练] | |
| §4.1.2 MLP_man(u_man) → s_man [可训练] | |
| §4.1.3 ResNet grid_sample → raw_deep [冻结] | |
| proj_deep(raw_deep) → h_deep [可训练] | |
| §4.1.4 W_g, MHCA, Transformer → s_seq [可训练] | |
| 训练时:optimizer 仅传入 model.trainable_parameters()。 | |
| """ | |
| from __future__ import annotations | |
| from dataclasses import dataclass, field | |
| from pathlib import Path | |
| from typing import Dict, List, Optional, Tuple, Union | |
| import numpy as np | |
| import torch | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| from elipse_feature import ( | |
| DEFAULT_IMG_SIZE, | |
| DEFAULT_N_THETA, | |
| EllipseFeatureBundle, | |
| FeaFields, | |
| FeaTensor, | |
| SequenceEllipseFeatures, | |
| extract_ellipse_features, | |
| preprocess_rgb_to_skeleton, | |
| unpack_fea_fields, | |
| ) | |
| from resnet_point_feature import default_resnet_checkpoint | |
| from convnext_point_feature import default_convnext_checkpoint | |
| PathLike = Union[str, Path, np.ndarray] | |
| U_MAN_DIM = 8 | |
| G_GLOBAL_DIM = 6 | |
| class Section41Config: | |
| n_theta: int = DEFAULT_N_THETA | |
| img_size: int = DEFAULT_IMG_SIZE | |
| d_geo: int = 32 | |
| d_man: int = 64 | |
| d_deep: int = 128 | |
| d_attn: int = 128 | |
| d_seq: int = 128 | |
| h_geo: int = 4 | |
| l_seq: int = 2 | |
| n_heads_seq: int = 4 | |
| ffn_dim: int = 256 | |
| patch_size: int = 7 | |
| mlp_geo_hidden: Tuple[int, ...] = (32,) | |
| mlp_man_hidden: Tuple[int, ...] = (64,) | |
| deep_backend: str = "convnext" | |
| resnet_checkpoint: Optional[str] = field(default_factory=default_resnet_checkpoint) | |
| convnext_checkpoint: Optional[str] = field(default_factory=default_convnext_checkpoint) | |
| class Section41Output: | |
| """§4.1 输出(张量字段在 eval 时可 .numpy())。""" | |
| prep: dict | |
| bundle: Optional[EllipseFeatureBundle] | |
| g_global: np.ndarray | |
| g_vec: np.ndarray | |
| u_man: np.ndarray | |
| s_man: np.ndarray | |
| h_deep: np.ndarray | |
| z: np.ndarray | |
| u_fuse: np.ndarray | |
| s_seq: np.ndarray | |
| config: Section41Config | |
| # --------------------------------------------------------------------------- | |
| # 工具 | |
| # --------------------------------------------------------------------------- | |
| def build_u_man(sequence: SequenceEllipseFeatures) -> np.ndarray: | |
| """u_man(t) = [Δr, Δθ, κ, τ, x, y, φ, valid]。""" | |
| valid = (1.0 - sequence.interp_flag).astype(np.float32) | |
| phi_norm = (sequence.theta_ideal / (2.0 * np.pi)).astype(np.float32) | |
| return np.concatenate( | |
| [ | |
| sequence.s_res[:, :4].astype(np.float32), | |
| sequence.s_base.astype(np.float32), | |
| phi_norm[:, None], | |
| valid[:, None], | |
| ], | |
| axis=1, | |
| ) | |
| def keypoints_px_from_u_man( | |
| u_man: Union[np.ndarray, torch.Tensor], | |
| half_norm: float = DEFAULT_IMG_SIZE / 2.0, | |
| ) -> np.ndarray: | |
| """ | |
| 由 u_man[:, 4:6](归一化 s_base)还原像素坐标 (x, y),供 §4.1.3 ResNet 采样。 | |
| half_norm 须与 DataLoader 中 extract_ellipse_features 一致(默认 50)。 | |
| """ | |
| if isinstance(u_man, torch.Tensor): | |
| s_base = u_man[..., 4:6].detach().cpu().numpy() | |
| else: | |
| s_base = np.asarray(u_man)[..., 4:6] | |
| return (s_base * half_norm).astype(np.float32) | |
| def keypoints_px_tensor( | |
| u_man: torch.Tensor, | |
| half_norm: float = DEFAULT_IMG_SIZE / 2.0, | |
| ) -> torch.Tensor: | |
| """u_man (..., 8) → 像素坐标 (..., 2),保留 device/dtype 供批量 ResNet 采样。""" | |
| kps = u_man[..., 4:6].float() * half_norm | |
| return kps | |
| def img_tensor_to_bgr( | |
| img_tensor: torch.Tensor, | |
| mean: Tuple[float, ...] = (0.485, 0.456, 0.406), | |
| std: Tuple[float, ...] = (0.229, 0.224, 0.225), | |
| ) -> np.ndarray: | |
| """DataLoader 输出的归一化 RGB (3,H,W) → BGR uint8,供 deep_branch 使用。""" | |
| import cv2 | |
| t = img_tensor.detach().cpu().float() | |
| if t.dim() != 3: | |
| raise ValueError(f"img_tensor 须为 (3,H,W),got {tuple(t.shape)}") | |
| for c in range(3): | |
| t[c] = t[c] * std[c] + mean[c] | |
| rgb = (t.permute(1, 2, 0).numpy() * 255.0).clip(0, 255).astype(np.uint8) | |
| return cv2.cvtColor(rgb, cv2.COLOR_RGB2BGR) | |
| def fea_fields_from_dataloader(fea_tensor: FeaTensor) -> FeaFields: | |
| """从 DataLoader 输出的 FeaTensor 反解各字段。""" | |
| if isinstance(fea_tensor, FeaTensor): | |
| return fea_tensor.fields | |
| return unpack_fea_fields(fea_tensor) | |
| def _build_mlp( | |
| in_dim: int, | |
| hidden: Tuple[int, ...], | |
| out_dim: int, | |
| ) -> nn.Sequential: | |
| layers: List[nn.Module] = [] | |
| dims = [in_dim, *hidden, out_dim] | |
| for i in range(len(dims) - 2): | |
| layers.extend([nn.Linear(dims[i], dims[i + 1]), nn.ReLU(inplace=True)]) | |
| layers.append(nn.Linear(dims[-2], dims[-1])) | |
| return nn.Sequential(*layers) | |
| def _freeze_module(module: nn.Module) -> None: | |
| module.eval() | |
| for p in module.parameters(): | |
| p.requires_grad = False | |
| # --------------------------------------------------------------------------- | |
| # §4.1.3 冻结深度分支 | |
| # --------------------------------------------------------------------------- | |
| class FrozenDeepBranch(nn.Module): | |
| """ | |
| §4.1.3 冻结视觉骨干:仅 ResNet(或 patch 裁剪)输出 raw_deep,不含可训练层。 | |
| 线性投影 proj_deep 在 Section41Encoder 中。 | |
| """ | |
| def __init__(self, config: Section41Config): | |
| super().__init__() | |
| from resnet_point_feature import RESNET_DEEP_RAW_DIM, ResNetFeatureExtractor | |
| from convnext_point_feature import CONVNEXT_DEEP_RAW_DIM, ConvNeXtFeatureExtractor | |
| self.backend = config.deep_backend.lower() | |
| if self.backend not in ("resnet", "convnext", "patch"): | |
| raise ValueError(f"deep_backend 需为 resnet、convnext 或 patch, got {self.backend}") | |
| if self.backend == "resnet": | |
| self.resnet = ResNetFeatureExtractor( | |
| checkpoint_path=config.resnet_checkpoint, | |
| ) | |
| self.convnext = None | |
| _freeze_module(self.resnet) | |
| self.raw_dim = RESNET_DEEP_RAW_DIM | |
| elif self.backend == "convnext": | |
| self.resnet = None | |
| self.convnext = ConvNeXtFeatureExtractor( | |
| checkpoint_path=config.convnext_checkpoint, | |
| ) | |
| _freeze_module(self.convnext) | |
| self.raw_dim = CONVNEXT_DEEP_RAW_DIM | |
| else: | |
| self.resnet = None | |
| self.convnext = None | |
| self.raw_dim = 3 * config.patch_size * config.patch_size | |
| def _extract_raw_resnet( | |
| self, | |
| rgb_bgr: np.ndarray, | |
| points_xy: np.ndarray, | |
| ) -> torch.Tensor: | |
| from resnet_point_feature import _rgb_bgr_to_tensor | |
| device = next(self.parameters()).device | |
| image_tensor, _, _ = _rgb_bgr_to_tensor(rgb_bgr, device) | |
| kps = torch.from_numpy(points_xy.astype(np.float32)).unsqueeze(0).to(device) | |
| _, _, h, w = image_tensor.shape | |
| kps[..., 0] = kps[..., 0].clamp(0, w - 1) | |
| kps[..., 1] = kps[..., 1].clamp(0, h - 1) | |
| return self.resnet(image_tensor, kps).squeeze(0) | |
| def _extract_raw_resnet_batch( | |
| self, | |
| images: torch.Tensor, | |
| keypoints: torch.Tensor, | |
| ) -> torch.Tensor: | |
| """images [B,3,H,W],keypoints [B,N,2] → raw_deep [B,N,raw_dim]。""" | |
| _, _, h, w = images.shape | |
| kps = keypoints.clone() | |
| kps[..., 0] = kps[..., 0].clamp(0, w - 1) | |
| kps[..., 1] = kps[..., 1].clamp(0, h - 1) | |
| return self.resnet(images, kps) | |
| def _extract_raw_convnext( | |
| self, | |
| rgb_bgr: np.ndarray, | |
| points_xy: np.ndarray, | |
| ) -> torch.Tensor: | |
| from resnet_point_feature import _rgb_bgr_to_tensor | |
| device = next(self.parameters()).device | |
| image_tensor, _, _ = _rgb_bgr_to_tensor(rgb_bgr, device) | |
| kps = torch.from_numpy(points_xy.astype(np.float32)).unsqueeze(0).to(device) | |
| _, _, h, w = image_tensor.shape | |
| kps[..., 0] = kps[..., 0].clamp(0, w - 1) | |
| kps[..., 1] = kps[..., 1].clamp(0, h - 1) | |
| return self.convnext(image_tensor, kps).squeeze(0) | |
| def _extract_raw_convnext_batch( | |
| self, | |
| images: torch.Tensor, | |
| keypoints: torch.Tensor, | |
| ) -> torch.Tensor: | |
| _, _, h, w = images.shape | |
| kps = keypoints.clone() | |
| kps[..., 0] = kps[..., 0].clamp(0, w - 1) | |
| kps[..., 1] = kps[..., 1].clamp(0, h - 1) | |
| return self.convnext(images, kps) | |
| def _extract_raw_patch( | |
| self, | |
| rgb_bgr: np.ndarray, | |
| points_xy: np.ndarray, | |
| patch_size: int, | |
| ) -> torch.Tensor: | |
| import cv2 | |
| h, w = rgb_bgr.shape[:2] | |
| r = patch_size // 2 | |
| rgb = cv2.cvtColor(rgb_bgr, cv2.COLOR_BGR2RGB).astype(np.float32) / 255.0 | |
| n = points_xy.shape[0] | |
| patches = [] | |
| for t in range(n): | |
| cx = int(round(points_xy[t, 0])) | |
| cy = int(round(points_xy[t, 1])) | |
| patch = np.zeros((patch_size, patch_size, 3), dtype=np.float32) | |
| for dy in range(-r, r + 1): | |
| for dx in range(-r, r + 1): | |
| y = int(np.clip(cy + dy, 0, h - 1)) | |
| x = int(np.clip(cx + dx, 0, w - 1)) | |
| patch[dy + r, dx + r] = rgb[y, x] | |
| patches.append(patch.reshape(-1)) | |
| return torch.from_numpy(np.stack(patches, axis=0)) | |
| def forward( | |
| self, | |
| rgb_bgr: np.ndarray, | |
| points_xy: np.ndarray, | |
| patch_size: int = 7, | |
| ) -> torch.Tensor: | |
| """返回 raw_deep: (N, raw_dim),无梯度。""" | |
| if self.backend == "resnet": | |
| return self._extract_raw_resnet(rgb_bgr, points_xy) | |
| if self.backend == "convnext": | |
| return self._extract_raw_convnext(rgb_bgr, points_xy) | |
| return self._extract_raw_patch(rgb_bgr, points_xy, patch_size) | |
| # --------------------------------------------------------------------------- | |
| # §4.1.4 可训练注意力与 Transformer | |
| # --------------------------------------------------------------------------- | |
| class GeoMultiHeadCrossAttention(nn.Module): | |
| """全局 q_global 广播,K/V 来自 s_man(论文 §4.1.4 MHCA)。""" | |
| def __init__(self, d_man: int, d_attn: int, n_heads: int): | |
| super().__init__() | |
| if d_attn % n_heads != 0: | |
| raise ValueError("d_attn 须能被 n_heads 整除") | |
| self.n_heads = n_heads | |
| self.d_head = d_attn // n_heads | |
| self.w_k = nn.Linear(d_man, d_attn, bias=False) | |
| self.w_v = nn.Linear(d_man, d_attn, bias=False) | |
| self.w_o = nn.Linear(d_attn, d_attn, bias=True) | |
| def forward(self, q_global: torch.Tensor, s_man: torch.Tensor) -> torch.Tensor: | |
| """ | |
| q_global: (d_attn,) 或 (B, d_attn) | |
| s_man: (N, d_man) 或 (B, N, d_man) | |
| 返回 z: (N, d_attn) 或 (B, N, d_attn) | |
| """ | |
| batched = q_global.dim() == 2 | |
| if not batched: | |
| q_global = q_global.unsqueeze(0) | |
| s_man = s_man.unsqueeze(0) | |
| b, n, _ = s_man.shape | |
| h, d = self.n_heads, self.d_head | |
| k = self.w_k(s_man).view(b, n, h, d) | |
| v = self.w_v(s_man).view(b, n, h, d) | |
| q = q_global.view(b, h, d) | |
| scale = d**0.5 | |
| scores = torch.einsum("bhd,bnhd->bhn", q, k) / scale | |
| attn = F.softmax(scores, dim=-1) | |
| z_heads = torch.einsum("bhn,bnhd->bhd", attn, v) | |
| z = z_heads.unsqueeze(1).expand(-1, n, -1, -1).reshape(b, n, h * d) | |
| z = self.w_o(z) | |
| if not batched: | |
| z = z.squeeze(0) | |
| return z | |
| class Section41Encoder(nn.Module): | |
| """ | |
| §4.1 完整编码器(nn.Module)。 | |
| 可训练:mlp_geo, mlp_man, proj_deep, w_g, mhca, fuse_norm, transformer, proj_seq | |
| 冻结:deep_branch(ResNet 骨干) | |
| """ | |
| def __init__(self, config: Optional[Section41Config] = None): | |
| super().__init__() | |
| self.config = config or Section41Config() | |
| cfg = self.config | |
| d_fuse = cfg.d_attn + cfg.d_deep | |
| self.mlp_geo = _build_mlp(G_GLOBAL_DIM, cfg.mlp_geo_hidden, cfg.d_geo) | |
| self.mlp_man = _build_mlp(U_MAN_DIM, cfg.mlp_man_hidden, cfg.d_man) | |
| self.w_g = nn.Linear(cfg.d_geo, cfg.d_attn, bias=False) | |
| self.mhca = GeoMultiHeadCrossAttention(cfg.d_man, cfg.d_attn, cfg.h_geo) | |
| self.deep_branch = FrozenDeepBranch(cfg) | |
| self.proj_deep = nn.Linear(self.deep_branch.raw_dim, cfg.d_deep, bias=True) | |
| self.fuse_norm = nn.LayerNorm(d_fuse) | |
| enc_layer = nn.TransformerEncoderLayer( | |
| d_model=d_fuse, | |
| nhead=cfg.n_heads_seq, | |
| dim_feedforward=cfg.ffn_dim, | |
| batch_first=True, | |
| norm_first=True, | |
| activation="gelu", | |
| ) | |
| self.transformer = nn.TransformerEncoder(enc_layer, num_layers=cfg.l_seq) | |
| self.proj_seq = nn.Linear(d_fuse, cfg.d_seq) | |
| def trainable_parameters(self) -> List[nn.Parameter]: | |
| """§4.1.1 / §4.1.2 / §4.1.3 proj_deep / §4.1.4 可训练参数。""" | |
| return [p for p in self.parameters() if p.requires_grad] | |
| def trainable_named_parameters(self) -> List[Tuple[str, nn.Parameter]]: | |
| return [(n, p) for n, p in self.named_parameters() if p.requires_grad] | |
| def frozen_parameters(self) -> List[nn.Parameter]: | |
| """仅冻结的 ResNet 骨干参数。""" | |
| return [p for p in self.deep_branch.parameters() if not p.requires_grad] | |
| def extract_h_deep( | |
| self, | |
| rgb_bgr: np.ndarray, | |
| points_xy: np.ndarray, | |
| ) -> torch.Tensor: | |
| """§4.1.3:冻结 backbone 采样 raw_deep,再经可训练 proj_deep → h_deep。""" | |
| with torch.no_grad(): | |
| raw = self.deep_branch( | |
| rgb_bgr, | |
| points_xy, | |
| patch_size=self.config.patch_size, | |
| ) | |
| return self.proj_deep(raw) | |
| def extract_h_deep_batch( | |
| self, | |
| img_tensor: torch.Tensor, | |
| u_man: torch.Tensor, | |
| half_norm: float = DEFAULT_IMG_SIZE / 2.0, | |
| ) -> torch.Tensor: | |
| """§4.1.3 批量:DataLoader 的 img_tensor [B,3,H,W] + u_man [B,N,8]。""" | |
| if self.deep_branch.backend not in ("resnet", "convnext"): | |
| raise NotImplementedError( | |
| "extract_h_deep_batch 仅支持 deep_backend=resnet 或 convnext" | |
| ) | |
| device = next(self.parameters()).device | |
| images = img_tensor.to(device) | |
| kps = keypoints_px_tensor(u_man, half_norm=half_norm).to(device) | |
| with torch.no_grad(): | |
| if self.deep_branch.backend == "resnet": | |
| raw = self.deep_branch._extract_raw_resnet_batch(images, kps) | |
| else: | |
| raw = self.deep_branch._extract_raw_convnext_batch(images, kps) | |
| return self.proj_deep(raw) | |
| def forward_tensors( | |
| self, | |
| g_global: torch.Tensor, | |
| u_man: torch.Tensor, | |
| h_deep: torch.Tensor, | |
| ) -> Dict[str, torch.Tensor]: | |
| """ | |
| 核心可微前向(训练用)。 | |
| g_global: (6,) 或 (B, 6) | |
| u_man: (N, 8) 或 (B, N, 8) | |
| h_deep: (N, d_deep) 或 (B, N, d_deep),由 extract_h_deep 产生(proj_deep 可反传) | |
| """ | |
| batched = g_global.dim() == 2 | |
| if not batched: | |
| g_global = g_global.unsqueeze(0) | |
| u_man = u_man.unsqueeze(0) | |
| h_deep = h_deep.unsqueeze(0) | |
| g_vec = self.mlp_geo(g_global) | |
| s_man = self.mlp_man(u_man) | |
| q_global = self.w_g(g_vec) | |
| z = self.mhca(q_global, s_man) | |
| u = torch.cat([z, h_deep], dim=-1) | |
| h = self.fuse_norm(u) | |
| h = self.transformer(h) | |
| s_seq = self.proj_seq(h) | |
| if not batched: | |
| s_seq = s_seq.squeeze(0) | |
| g_vec = g_vec.squeeze(0) | |
| s_man = s_man.squeeze(0) | |
| z = z.squeeze(0) | |
| u = u.squeeze(0) | |
| return { | |
| "g_vec": g_vec, | |
| "s_man": s_man, | |
| "z": z, | |
| "u_fuse": u, | |
| "s_seq": s_seq, | |
| } | |
| def _compute_geometry( | |
| self, | |
| rgb: PathLike, | |
| ) -> Tuple[dict, EllipseFeatureBundle, np.ndarray, np.ndarray, np.ndarray]: | |
| cfg = self.config | |
| prep = preprocess_rgb_to_skeleton(rgb, size=cfg.img_size) | |
| hn = cfg.img_size / 2.0 | |
| bundle = extract_ellipse_features( | |
| prep["skeleton"], | |
| n_theta=cfg.n_theta, | |
| half_norm=hn, | |
| ) | |
| g_global = bundle.global_feat.as_vector() | |
| u_man = build_u_man(bundle.sequence) | |
| return prep, bundle, g_global, u_man, bundle.sequence.s_base_px | |
| def forward( | |
| self, | |
| rgb: PathLike, | |
| *, | |
| return_numpy: bool = True, | |
| ) -> Section41Output: | |
| """单张图完整 §4.1 流程。""" | |
| prep, bundle, g_global, u_man, pts = self._compute_geometry(rgb) | |
| device = next(self.parameters()).device | |
| g_t = torch.from_numpy(g_global).float().to(device) | |
| u_t = torch.from_numpy(u_man).float().to(device) | |
| h_deep = self.extract_h_deep(prep["rgb"], pts) | |
| out_t = self.forward_tensors(g_t, u_t, h_deep) | |
| def _np(x: torch.Tensor) -> np.ndarray: | |
| return x.detach().cpu().numpy() | |
| if return_numpy: | |
| return Section41Output( | |
| prep=prep, | |
| bundle=bundle, | |
| g_global=g_global, | |
| g_vec=_np(out_t["g_vec"]), | |
| u_man=u_man, | |
| s_man=_np(out_t["s_man"]), | |
| h_deep=_np(h_deep), | |
| z=_np(out_t["z"]), | |
| u_fuse=_np(out_t["u_fuse"]), | |
| s_seq=_np(out_t["s_seq"]), | |
| config=self.config, | |
| ) | |
| raise NotImplementedError("return_numpy=False 时可扩展返回张量字典") | |
| def forward_from_fea( | |
| self, | |
| fea_tensor: FeaTensor, | |
| img_bgr: Optional[np.ndarray] = None, | |
| img_tensor: Optional[torch.Tensor] = None, | |
| half_norm: float = DEFAULT_IMG_SIZE / 2.0, | |
| return_numpy: bool = True, | |
| ) -> Union[Section41Output, Dict[str, torch.Tensor]]: | |
| """ | |
| 使用 DataLoader 输出的 fea_tensor 运行 §4.1。 | |
| Parameters | |
| ---------- | |
| fea_tensor : FeaTensor,含 g_global / u_man 等(列向量打包) | |
| img_bgr : 与特征同坐标系的 BGR 图;与 img_tensor 二选一 | |
| img_tensor : DataLoader 返回的归一化 RGB (3,H,W) | |
| half_norm : u_man 中 s_base 的归一化半宽(默认 50,对应 100×100 图) | |
| """ | |
| if img_bgr is None: | |
| if img_tensor is None: | |
| raise ValueError("forward_from_fea 需要 img_bgr 或 img_tensor") | |
| img_bgr = img_tensor_to_bgr(img_tensor) | |
| fields = fea_fields_from_dataloader(fea_tensor) | |
| if fields.u_man.dim() == 3: | |
| raise NotImplementedError("batch FeaTensor 请逐样本调用 forward_from_fea") | |
| g_global = fields.g_global.detach().cpu().numpy().astype(np.float32) | |
| u_man = fields.u_man.detach().cpu().numpy().astype(np.float32) | |
| points_px = keypoints_px_from_u_man(u_man, half_norm=half_norm) | |
| device = next(self.parameters()).device | |
| g_t = torch.from_numpy(g_global).float().to(device) | |
| u_t = torch.from_numpy(u_man).float().to(device) | |
| h_deep = self.extract_h_deep( | |
| img_bgr, | |
| points_px, | |
| ) | |
| out_t = self.forward_tensors(g_t, u_t, h_deep) | |
| if not return_numpy: | |
| out_t["g_global"] = g_t | |
| out_t["u_man"] = u_t | |
| out_t["h_deep"] = h_deep | |
| out_t["points_px"] = torch.from_numpy(points_px) | |
| return out_t | |
| def _np(x: torch.Tensor) -> np.ndarray: | |
| return x.detach().cpu().numpy() | |
| return Section41Output( | |
| prep={"rgb": img_bgr}, | |
| bundle=None, | |
| g_global=g_global, | |
| g_vec=_np(out_t["g_vec"]), | |
| u_man=u_man, | |
| s_man=_np(out_t["s_man"]), | |
| h_deep=_np(h_deep), | |
| z=_np(out_t["z"]), | |
| u_fuse=_np(out_t["u_fuse"]), | |
| s_seq=_np(out_t["s_seq"]), | |
| config=self.config, | |
| ) | |
| def save_checkpoint(self, path: Union[str, Path], extra: Optional[dict] = None) -> None: | |
| payload = { | |
| "config": self.config, | |
| "state_dict": self.state_dict(), | |
| "extra": extra or {}, | |
| } | |
| torch.save(payload, path) | |
| def load_checkpoint( | |
| cls, | |
| path: Union[str, Path], | |
| map_location: Optional[str] = None, | |
| strict: bool = True, | |
| ) -> "Section41Encoder": | |
| payload = torch.load(path, map_location=map_location, weights_only=False) | |
| cfg = payload.get("config") or Section41Config() | |
| if not isinstance(cfg, Section41Config): | |
| cfg = Section41Config(**cfg) if isinstance(cfg, dict) else Section41Config() | |
| model = cls(cfg) | |
| model.load_state_dict(payload["state_dict"], strict=strict) | |
| return model | |
| def extract_section41( | |
| rgb: PathLike, | |
| config: Optional[Section41Config] = None, | |
| encoder: Optional[Section41Encoder] = None, | |
| device: Optional[Union[str, torch.device]] = None, | |
| ) -> Section41Output: | |
| enc = encoder or Section41Encoder(config) | |
| if device is not None: | |
| enc = enc.to(device) | |
| enc.eval() | |
| return enc.forward(rgb) | |
| def demo( | |
| image_path: str = "data/images/20938.png", | |
| config: Optional[Section41Config] = None, | |
| device: str = "cpu", | |
| ) -> Section41Output: | |
| enc = Section41Encoder(config).to(device) | |
| n_trainable = sum(p.numel() for p in enc.trainable_parameters()) | |
| n_frozen = sum(p.numel() for p in enc.frozen_parameters()) | |
| n_total = sum(p.numel() for p in enc.parameters()) | |
| out = enc.forward(image_path) | |
| c = out.config | |
| print(f"§4.1 demo: {image_path} [deep={enc.deep_branch.backend}]") | |
| print(f" 可训练参数: {n_trainable:,} / 冻结: {n_frozen:,} / 总: {n_total:,}") | |
| print(f" proj_deep trainable: {enc.proj_deep.weight.requires_grad}") | |
| print(f" g_vec {out.g_vec.shape}") | |
| print(f" u_man {out.u_man.shape}") | |
| print(f" s_man {out.s_man.shape}") | |
| print(f" h_deep {out.h_deep.shape}") | |
| print(f" z (MHCA) {out.z.shape}") | |
| print(f" s_seq {out.s_seq.shape}") | |
| if out.bundle is not None: | |
| print(f" interp {int(out.bundle.sequence.interp_flag.sum())} / {c.n_theta}") | |
| return out | |
| def demo_from_dataloader( | |
| sample_ratio: float = 0.001, | |
| batch_size: int = 2, | |
| half_norm: float = DEFAULT_IMG_SIZE / 2.0, | |
| config: Optional[Section41Config] = None, | |
| device: str = "cpu", | |
| ) -> Section41Output: | |
| """ | |
| 从 exp6_data_loader 取样本,用 fea_tensor(g_global + u_man + 还原采样点)调试 §4.1。 | |
| """ | |
| from itertools import islice | |
| from exp6_data_loader import Config as DataConfig, create_dataloaders | |
| cfg = config or Section41Config() | |
| enc = Section41Encoder(cfg).to(device) | |
| enc.eval() | |
| train_loader, _, _, _ = create_dataloaders( | |
| batch_size=batch_size, | |
| num_workers=0, | |
| sample_ratio=sample_ratio, | |
| verbose=False, | |
| ) | |
| batch = next(iter(train_loader)) | |
| input_all, labels, pen_ids, writers, image_ids = batch | |
| fea_tensor, ske_tensor, img_tensor = input_all | |
| print("[section41] dataloader batch") | |
| print(f" fea_tensor.data: {fea_tensor.data.shape}") | |
| print(f" img_tensor: {tuple(img_tensor.shape)} label: {labels.tolist()}") | |
| # 取 batch 中第一个样本 | |
| fea_one = FeaTensor(data=fea_tensor.data[0]) | |
| img_one = img_tensor[0] | |
| fields = fea_one.fields | |
| points_px = keypoints_px_from_u_man(fields.u_man, half_norm=half_norm) | |
| print(f" g_global: {tuple(fields.g_global.shape)}") | |
| print(f" u_man: {tuple(fields.u_man.shape)}") | |
| print(f" s_res: {tuple(fields.s_res.shape)} valid sum: {fields.valid.sum().item():.0f}") | |
| print(f" points_px:{points_px.shape} range x=[{points_px[:,0].min():.1f},{points_px[:,0].max():.1f}] " | |
| f"y=[{points_px[:,1].min():.1f},{points_px[:,1].max():.1f}]") | |
| out = enc.forward_from_fea( | |
| fea_one, | |
| img_tensor=img_one, | |
| half_norm=half_norm, | |
| ) | |
| n_trainable = sum(p.numel() for p in enc.trainable_parameters()) | |
| print("[section41] forward_from_fea") | |
| print(f" writer: {writers[0]} image_id: {image_ids[0]}") | |
| print(f" trainable params: {n_trainable:,}") | |
| print(f" g_vec: {out.g_vec.shape}") | |
| print(f" s_man: {out.s_man.shape}") | |
| print(f" h_deep: {out.h_deep.shape}") | |
| print(f" z: {out.z.shape}") | |
| print(f" s_seq: {out.s_seq.shape}") | |
| # 批量 smoke test(逐样本 deep + batched forward_tensors 的可训练部分) | |
| print("[section41] batch smoke (trainable path only)") | |
| fields_b = fea_tensor.fields | |
| g_b = fields_b.g_global.float().to(device) | |
| u_b = fields_b.u_man.float().to(device) | |
| h_list = [] | |
| for i in range(img_tensor.shape[0]): | |
| pts = keypoints_px_from_u_man(u_b[i], half_norm=half_norm) | |
| bgr = img_tensor_to_bgr(img_tensor[i]) | |
| h_list.append(enc.extract_h_deep(bgr, pts)) | |
| h_b = torch.stack(h_list, dim=0) | |
| out_b = enc.forward_tensors(g_b, u_b, h_b) | |
| print(f" batch s_seq: {tuple(out_b['s_seq'].shape)}") | |
| return out | |
| def main() -> None: | |
| import argparse | |
| p = argparse.ArgumentParser(description="§4.1 ellipse sequence encoder (PyTorch)") | |
| p.add_argument("image", nargs="?", default="data/images/20938.png") | |
| p.add_argument("--save", type=str, default=None, help="保存输出 .npz") | |
| p.add_argument("--ckpt", type=str, default=None, help="保存可训练权重 .pth") | |
| p.add_argument( | |
| "--deep", | |
| choices=("convnext", "resnet", "patch"), | |
| default="convnext", | |
| ) | |
| p.add_argument( | |
| "--resnet-ckpt", | |
| type=str, | |
| default=None, | |
| help=f"ResNet 权重(默认 {default_resnet_checkpoint()})", | |
| ) | |
| p.add_argument( | |
| "--convnext-ckpt", | |
| type=str, | |
| default=None, | |
| help=f"ConvNeXt 权重(默认 {default_convnext_checkpoint()})", | |
| ) | |
| p.add_argument("--device", default="cpu") | |
| p.add_argument( | |
| "--from-dataloader", | |
| action="store_true", | |
| help="使用 exp6_data_loader 输出的 fea_tensor 调试 §4.1", | |
| ) | |
| p.add_argument("--sample-ratio", type=float, default=0.001) | |
| p.add_argument("--batch-size", type=int, default=2) | |
| args = p.parse_args() | |
| cfg = Section41Config( | |
| deep_backend=args.deep, | |
| resnet_checkpoint=args.resnet_ckpt or default_resnet_checkpoint(), | |
| convnext_checkpoint=args.convnext_ckpt or default_convnext_checkpoint(), | |
| ) | |
| if args.from_dataloader: | |
| demo_from_dataloader( | |
| sample_ratio=args.sample_ratio, | |
| batch_size=args.batch_size, | |
| config=cfg, | |
| device=args.device, | |
| ) | |
| return | |
| enc = Section41Encoder(cfg).to(args.device) | |
| out = demo(args.image, config=cfg, device=args.device) | |
| if args.ckpt: | |
| enc.save_checkpoint(args.ckpt) | |
| print("checkpoint:", args.ckpt) | |
| if args.save: | |
| np.savez( | |
| args.save, | |
| g_vec=out.g_vec, | |
| s_man=out.s_man, | |
| h_deep=out.h_deep, | |
| z=out.z, | |
| s_seq=out.s_seq, | |
| u_man=out.u_man, | |
| s_res=out.bundle.sequence.s_res, | |
| ) | |
| print("saved:", args.save) | |
| if __name__ == "__main__": | |
| demo_from_dataloader() | |