Download lib/models/decoder/decoder_hamer_style.py from dqj5182/FECO: direct link, hf CLI and curl.
- Browser
- Download file 14.4 kB
-
https://huggingface.co/spaces/dqj5182/FECO/resolve/main/lib/models/decoder/decoder_hamer_style.py
- Command line
-
hf download hf://spaces/dqj5182/FECO/lib/models/decoder/decoder_hamer_style.py
-
curl -L -o decoder_hamer_style.py https://huggingface.co/spaces/dqj5182/FECO/resolve/main/lib/models/decoder/decoder_hamer_style.py
14.4 kB
| import pickle | |
| import numpy as np | |
| from einops import rearrange | |
| from inspect import isfunction | |
| from typing import Callable, Optional | |
| import torch | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| from lib.core.config import cfg | |
| J_regressor_foot = np.load(cfg.MODEL.J_regressor_foot_path) | |
| J_regressor_foot_openpose = np.load(cfg.MODEL.J_regressor_foot_openpose_path) | |
| # This function is from HaMeR (https://github.com/geopavlakos/hamer). | |
| def exists(val): | |
| return val is not None | |
| # This function is from HaMeR (https://github.com/geopavlakos/hamer). | |
| def default(val, d): | |
| if exists(val): | |
| return val | |
| return d() if isfunction(d) else d | |
| # This class is from HaMeR (https://github.com/geopavlakos/hamer). | |
| class Attention(nn.Module): | |
| def __init__(self, dim, heads=8, dim_head=64, dropout=0.0): | |
| super().__init__() | |
| inner_dim = dim_head * heads | |
| project_out = not (heads == 1 and dim_head == dim) | |
| self.heads = heads | |
| self.scale = dim_head**-0.5 | |
| self.attend = nn.Softmax(dim=-1) | |
| self.dropout = nn.Dropout(dropout) | |
| self.to_qkv = nn.Linear(dim, inner_dim * 3, bias=False) | |
| self.to_out = ( | |
| nn.Sequential(nn.Linear(inner_dim, dim), nn.Dropout(dropout)) | |
| if project_out | |
| else nn.Identity() | |
| ) | |
| def forward(self, x): | |
| qkv = self.to_qkv(x).chunk(3, dim=-1) | |
| q, k, v = map(lambda t: rearrange(t, "b n (h d) -> b h n d", h=self.heads), qkv) | |
| dots = torch.matmul(q, k.transpose(-1, -2)) * self.scale | |
| attn = self.attend(dots) | |
| attn = self.dropout(attn) | |
| out = torch.matmul(attn, v) | |
| out = rearrange(out, "b h n d -> b n (h d)") | |
| return self.to_out(out) | |
| # This class is from HaMeR (https://github.com/geopavlakos/hamer). | |
| class CrossAttention(nn.Module): | |
| def __init__(self, dim, context_dim=None, heads=8, dim_head=64, dropout=0.0): | |
| super().__init__() | |
| inner_dim = dim_head * heads | |
| project_out = not (heads == 1 and dim_head == dim) | |
| self.heads = heads | |
| self.scale = dim_head**-0.5 | |
| self.attend = nn.Softmax(dim=-1) | |
| self.dropout = nn.Dropout(dropout) | |
| context_dim = default(context_dim, dim) | |
| self.to_kv = nn.Linear(context_dim, inner_dim * 2, bias=False) | |
| self.to_q = nn.Linear(dim, inner_dim, bias=False) | |
| self.to_out = ( | |
| nn.Sequential(nn.Linear(inner_dim, dim), nn.Dropout(dropout)) | |
| if project_out | |
| else nn.Identity() | |
| ) | |
| def forward(self, x, context=None): | |
| context = default(context, x) | |
| k, v = self.to_kv(context).chunk(2, dim=-1) | |
| q = self.to_q(x) | |
| q, k, v = map(lambda t: rearrange(t, "b n (h d) -> b h n d", h=self.heads), [q, k, v]) | |
| dots = torch.matmul(q, k.transpose(-1, -2)) * self.scale | |
| attn = self.attend(dots) | |
| attn = self.dropout(attn) | |
| out = torch.matmul(attn, v) | |
| out = rearrange(out, "b h n d -> b n (h d)") | |
| return self.to_out(out) | |
| # This class is from HaMeR (https://github.com/geopavlakos/hamer). | |
| class FeedForward(nn.Module): | |
| def __init__(self, dim, hidden_dim, dropout=0.0): | |
| super().__init__() | |
| self.net = nn.Sequential( | |
| nn.Linear(dim, hidden_dim), | |
| nn.GELU(), | |
| nn.Dropout(dropout), | |
| nn.Linear(hidden_dim, dim), | |
| nn.Dropout(dropout), | |
| ) | |
| def forward(self, x): | |
| return self.net(x) | |
| # This class is from HaMeR (https://github.com/geopavlakos/hamer). | |
| class Transformer(nn.Module): | |
| def __init__( | |
| self, | |
| dim: int, | |
| depth: int, | |
| heads: int, | |
| dim_head: int, | |
| mlp_dim: int, | |
| dropout: float = 0.0, | |
| norm: str = "layer", | |
| norm_cond_dim: int = -1, | |
| ): | |
| super().__init__() | |
| self.layers = nn.ModuleList([]) | |
| for _ in range(depth): | |
| sa = Attention(dim, heads=heads, dim_head=dim_head, dropout=dropout) | |
| ff = FeedForward(dim, mlp_dim, dropout=dropout) | |
| self.layers.append( | |
| nn.ModuleList( | |
| [ | |
| PreNorm(dim, sa, norm=norm, norm_cond_dim=norm_cond_dim), | |
| PreNorm(dim, ff, norm=norm, norm_cond_dim=norm_cond_dim), | |
| ] | |
| ) | |
| ) | |
| def forward(self, x: torch.Tensor, *args): | |
| for attn, ff in self.layers: | |
| x = attn(x, *args) + x | |
| x = ff(x, *args) + x | |
| return x | |
| class AdaptiveLayerNorm1D(torch.nn.Module): | |
| def __init__(self, data_dim: int, norm_cond_dim: int): | |
| super().__init__() | |
| if data_dim <= 0: | |
| raise ValueError(f"data_dim must be positive, but got {data_dim}") | |
| if norm_cond_dim <= 0: | |
| raise ValueError(f"norm_cond_dim must be positive, but got {norm_cond_dim}") | |
| self.norm = torch.nn.LayerNorm( | |
| data_dim | |
| ) | |
| self.linear = torch.nn.Linear(norm_cond_dim, 2 * data_dim) | |
| torch.nn.init.zeros_(self.linear.weight) | |
| torch.nn.init.zeros_(self.linear.bias) | |
| def forward(self, x: torch.Tensor, t: torch.Tensor) -> torch.Tensor: | |
| x = self.norm(x) | |
| alpha, beta = self.linear(t).chunk(2, dim=-1) | |
| if x.dim() > 2: | |
| alpha = alpha.view(alpha.shape[0], *([1] * (x.dim() - 2)), alpha.shape[1]) | |
| beta = beta.view(beta.shape[0], *([1] * (x.dim() - 2)), beta.shape[1]) | |
| return x * (1 + alpha) + beta | |
| def normalization_layer(norm: Optional[str], dim: int, norm_cond_dim: int = -1): | |
| if norm == "batch": | |
| return torch.nn.BatchNorm1d(dim) | |
| elif norm == "layer": | |
| return torch.nn.LayerNorm(dim) | |
| elif norm == "ada": | |
| assert norm_cond_dim > 0, f"norm_cond_dim must be positive, got {norm_cond_dim}" | |
| return AdaptiveLayerNorm1D(dim, norm_cond_dim) | |
| elif norm is None: | |
| return torch.nn.Identity() | |
| else: | |
| raise ValueError(f"Unknown norm: {norm}") | |
| class PreNorm(nn.Module): | |
| def __init__(self, dim: int, fn: Callable, norm: str = "layer", norm_cond_dim: int = -1): | |
| super().__init__() | |
| self.norm = normalization_layer(norm, dim, norm_cond_dim) | |
| self.fn = fn | |
| def forward(self, x: torch.Tensor, *args, **kwargs): | |
| if isinstance(self.norm, AdaptiveLayerNorm1D): | |
| return self.fn(self.norm(x, *args), **kwargs) | |
| else: | |
| return self.fn(self.norm(x), **kwargs) | |
| # This class is from HaMeR (https://github.com/geopavlakos/hamer). | |
| class TransformerCrossAttn(nn.Module): | |
| def __init__( | |
| self, | |
| dim: int, | |
| depth: int, | |
| heads: int, | |
| dim_head: int, | |
| mlp_dim: int, | |
| dropout: float = 0.0, | |
| norm: str = "layer", | |
| norm_cond_dim: int = -1, | |
| context_dim: Optional[int] = None, | |
| ): | |
| super().__init__() | |
| self.layers = nn.ModuleList([]) | |
| for _ in range(depth): | |
| sa = Attention(dim, heads=heads, dim_head=dim_head, dropout=dropout) | |
| ca = CrossAttention( | |
| dim, context_dim=context_dim, heads=heads, dim_head=dim_head, dropout=dropout | |
| ) | |
| ff = FeedForward(dim, mlp_dim, dropout=dropout) | |
| self.layers.append( | |
| nn.ModuleList( | |
| [ | |
| PreNorm(dim, sa, norm=norm, norm_cond_dim=norm_cond_dim), | |
| PreNorm(dim, ca, norm=norm, norm_cond_dim=norm_cond_dim), | |
| PreNorm(dim, ff, norm=norm, norm_cond_dim=norm_cond_dim), | |
| ] | |
| ) | |
| ) | |
| def forward(self, x: torch.Tensor, *args, context=None, context_list=None): | |
| if context_list is None: | |
| context_list = [context] * len(self.layers) | |
| if len(context_list) != len(self.layers): | |
| raise ValueError(f"len(context_list) != len(self.layers) ({len(context_list)} != {len(self.layers)})") | |
| for i, (self_attn, cross_attn, ff) in enumerate(self.layers): | |
| x = self_attn(x, *args) + x | |
| x = cross_attn(x, *args, context=context_list[i]) + x | |
| x = ff(x, *args) + x | |
| return x | |
| # This class is from HaMeR (https://github.com/geopavlakos/hamer). | |
| class DropTokenDropout(nn.Module): | |
| def __init__(self, p: float = 0.1): | |
| super().__init__() | |
| if p < 0 or p > 1: | |
| raise ValueError( | |
| "dropout probability has to be between 0 and 1, " "but got {}".format(p) | |
| ) | |
| self.p = p | |
| def forward(self, x: torch.Tensor): | |
| if self.training and self.p > 0: | |
| zero_mask = torch.full_like(x[0, :, 0], self.p).bernoulli().bool() | |
| if zero_mask.any(): | |
| x = x[:, ~zero_mask, :] | |
| return x | |
| # This class is from HaMeR (https://github.com/geopavlakos/hamer). | |
| class ZeroTokenDropout(nn.Module): | |
| def __init__(self, p: float = 0.1): | |
| super().__init__() | |
| if p < 0 or p > 1: | |
| raise ValueError( | |
| "dropout probability has to be between 0 and 1, " "but got {}".format(p) | |
| ) | |
| self.p = p | |
| def forward(self, x: torch.Tensor): | |
| # x: (batch_size, seq_len, dim) | |
| if self.training and self.p > 0: | |
| zero_mask = torch.full_like(x[:, :, 0], self.p).bernoulli().bool() | |
| # Zero-out the masked tokens | |
| x[zero_mask, :] = 0 | |
| return x | |
| # This class is from HaMeR (https://github.com/geopavlakos/hamer). | |
| class TransformerDecoder(nn.Module): | |
| def __init__( | |
| self, | |
| num_tokens: int, | |
| token_dim: int, | |
| dim: int, | |
| depth: int, | |
| heads: int, | |
| mlp_dim: int, | |
| dim_head: int = 64, | |
| dropout: float = 0.0, | |
| emb_dropout: float = 0.0, | |
| emb_dropout_type: str = 'drop', | |
| norm: str = "layer", | |
| norm_cond_dim: int = -1, | |
| context_dim: Optional[int] = None, | |
| skip_token_embedding: bool = False, | |
| ): | |
| super().__init__() | |
| if not skip_token_embedding: | |
| self.to_token_embedding = nn.Linear(token_dim, dim) | |
| else: | |
| self.to_token_embedding = nn.Identity() | |
| if token_dim != dim: | |
| raise ValueError( | |
| f"token_dim ({token_dim}) != dim ({dim}) when skip_token_embedding is True" | |
| ) | |
| self.pos_embedding = nn.Parameter(torch.randn(1, num_tokens, dim)) | |
| if emb_dropout_type == "drop": | |
| self.dropout = DropTokenDropout(emb_dropout) | |
| elif emb_dropout_type == "zero": | |
| self.dropout = ZeroTokenDropout(emb_dropout) | |
| elif emb_dropout_type == "normal": | |
| self.dropout = nn.Dropout(emb_dropout) | |
| self.transformer = TransformerCrossAttn( | |
| dim, | |
| depth, | |
| heads, | |
| dim_head, | |
| mlp_dim, | |
| dropout, | |
| norm=norm, | |
| norm_cond_dim=norm_cond_dim, | |
| context_dim=context_dim, | |
| ) | |
| def forward(self, inp: torch.Tensor, *args, context=None, context_list=None): | |
| x = self.to_token_embedding(inp) | |
| b, n, _ = x.shape | |
| x = self.dropout(x) | |
| x = x + self.pos_embedding[:, :n] | |
| x = self.transformer(x, *args, context=context, context_list=context_list) | |
| return x | |
| def rot6d_to_rotmat(x: torch.Tensor) -> torch.Tensor: | |
| x = x.reshape(-1,2,3).permute(0, 2, 1).contiguous() | |
| a1 = x[:, :, 0] | |
| a2 = x[:, :, 1] | |
| b1 = F.normalize(a1) | |
| b2 = F.normalize(a2 - torch.einsum('bi,bi->b', b1, a2).unsqueeze(-1) * b1) | |
| b3 = torch.cross(b1, b2) | |
| return torch.stack((b1, b2, b3), dim=-1) | |
| def aa_to_rotmat(theta: torch.Tensor): | |
| norm = torch.norm(theta + 1e-8, p = 2, dim = 1) | |
| angle = torch.unsqueeze(norm, -1) | |
| normalized = torch.div(theta, angle) | |
| angle = angle * 0.5 | |
| v_cos = torch.cos(angle) | |
| v_sin = torch.sin(angle) | |
| quat = torch.cat([v_cos, v_sin * normalized], dim = 1) | |
| return quat_to_rotmat(quat) | |
| # This module is modified from MANOTransformerDecoderHead of HaMeR (https://github.com/geopavlakos/hamer). All cfg are directly initialized. | |
| class ContactTransformerDecoderHead(nn.Module): | |
| """ Cross-attention based MANO Transformer decoder | |
| """ | |
| def __init__(self): | |
| super().__init__() | |
| transformer_args = dict( | |
| num_tokens=1, | |
| token_dim=1, | |
| dim=1024, | |
| ) | |
| if cfg.MODEL.backbone_type in ['resnet-50', 'resnet-101', 'resnet-152', 'hrnet-w32', 'hrnet-w48']: | |
| context_dim = 2048 | |
| elif cfg.MODEL.backbone_type in ['vit-h-14']: | |
| context_dim = 1280 | |
| elif cfg.MODEL.backbone_type in ['vit-l-16']: | |
| context_dim = 1024 | |
| elif cfg.MODEL.backbone_type in ['vit-b-16']: | |
| context_dim = 768 | |
| elif cfg.MODEL.backbone_type in ['resnet-18', 'resnet-34']: | |
| context_dim = 512 | |
| elif cfg.MODEL.backbone_type in ['vit-s-16']: | |
| context_dim = 384 | |
| else: | |
| raise NotImplementedError | |
| TRANSFORMER_DECODER_CONFIG = {'depth': 6, 'heads': 8, 'mlp_dim': 1024, 'dim_head': 64, 'dropout': 0.0, 'emb_dropout': 0.0, 'norm': 'layer', 'context_dim': context_dim} | |
| transformer_args.update(dict(TRANSFORMER_DECODER_CONFIG)) | |
| self.transformer = TransformerDecoder( | |
| **transformer_args | |
| ) | |
| self.deccontact = nn.Linear(1024, 265) # 265 is number of foot vertices | |
| self.init_contact = nn.Parameter(torch.randn(1, 265, requires_grad=True)) | |
| def forward(self, x, **kwargs): | |
| batch_size = x.shape[0] | |
| device = x.device | |
| x = rearrange(x, 'b c h w -> b (h w) c') | |
| init_contact = self.init_contact.expand(batch_size, -1) | |
| pred_contact = init_contact | |
| token = torch.zeros(batch_size, 1, 1, device=x.device) | |
| token_out = self.transformer(token, context=x) | |
| token_out = token_out[:, 0] | |
| pred_contact = self.deccontact(token_out) + pred_contact | |
| pred_joint_contact = (torch.tensor(J_regressor_foot, dtype=torch.float32, device=device) @ pred_contact.T).T | |
| pred_joint_contact_openpose = (torch.tensor(J_regressor_foot_openpose, dtype=torch.float32, device=device) @ pred_contact.T).T | |
| pred_per_foot_contact = torch.logsumexp(pred_contact, dim=1) | |
| return pred_contact, pred_joint_contact, pred_joint_contact_openpose, pred_per_foot_contact | |