import pickle import numpy as np from einops import rearrange from inspect import isfunction from typing import Callable, Optional import torch import torch.nn as nn import torch.nn.functional as F from lib.core.config import cfg J_regressor_foot = np.load(cfg.MODEL.J_regressor_foot_path) J_regressor_foot_openpose = np.load(cfg.MODEL.J_regressor_foot_openpose_path) # This function is from HaMeR (https://github.com/geopavlakos/hamer). def exists(val): return val is not None # This function is from HaMeR (https://github.com/geopavlakos/hamer). def default(val, d): if exists(val): return val return d() if isfunction(d) else d # This class is from HaMeR (https://github.com/geopavlakos/hamer). class Attention(nn.Module): def __init__(self, dim, heads=8, dim_head=64, dropout=0.0): super().__init__() inner_dim = dim_head * heads project_out = not (heads == 1 and dim_head == dim) self.heads = heads self.scale = dim_head**-0.5 self.attend = nn.Softmax(dim=-1) self.dropout = nn.Dropout(dropout) self.to_qkv = nn.Linear(dim, inner_dim * 3, bias=False) self.to_out = ( nn.Sequential(nn.Linear(inner_dim, dim), nn.Dropout(dropout)) if project_out else nn.Identity() ) def forward(self, x): qkv = self.to_qkv(x).chunk(3, dim=-1) q, k, v = map(lambda t: rearrange(t, "b n (h d) -> b h n d", h=self.heads), qkv) dots = torch.matmul(q, k.transpose(-1, -2)) * self.scale attn = self.attend(dots) attn = self.dropout(attn) out = torch.matmul(attn, v) out = rearrange(out, "b h n d -> b n (h d)") return self.to_out(out) # This class is from HaMeR (https://github.com/geopavlakos/hamer). class CrossAttention(nn.Module): def __init__(self, dim, context_dim=None, heads=8, dim_head=64, dropout=0.0): super().__init__() inner_dim = dim_head * heads project_out = not (heads == 1 and dim_head == dim) self.heads = heads self.scale = dim_head**-0.5 self.attend = nn.Softmax(dim=-1) self.dropout = nn.Dropout(dropout) context_dim = default(context_dim, dim) self.to_kv = nn.Linear(context_dim, inner_dim * 2, bias=False) self.to_q = nn.Linear(dim, inner_dim, bias=False) self.to_out = ( nn.Sequential(nn.Linear(inner_dim, dim), nn.Dropout(dropout)) if project_out else nn.Identity() ) def forward(self, x, context=None): context = default(context, x) k, v = self.to_kv(context).chunk(2, dim=-1) q = self.to_q(x) q, k, v = map(lambda t: rearrange(t, "b n (h d) -> b h n d", h=self.heads), [q, k, v]) dots = torch.matmul(q, k.transpose(-1, -2)) * self.scale attn = self.attend(dots) attn = self.dropout(attn) out = torch.matmul(attn, v) out = rearrange(out, "b h n d -> b n (h d)") return self.to_out(out) # This class is from HaMeR (https://github.com/geopavlakos/hamer). class FeedForward(nn.Module): def __init__(self, dim, hidden_dim, dropout=0.0): super().__init__() self.net = nn.Sequential( nn.Linear(dim, hidden_dim), nn.GELU(), nn.Dropout(dropout), nn.Linear(hidden_dim, dim), nn.Dropout(dropout), ) def forward(self, x): return self.net(x) # This class is from HaMeR (https://github.com/geopavlakos/hamer). class Transformer(nn.Module): def __init__( self, dim: int, depth: int, heads: int, dim_head: int, mlp_dim: int, dropout: float = 0.0, norm: str = "layer", norm_cond_dim: int = -1, ): super().__init__() self.layers = nn.ModuleList([]) for _ in range(depth): sa = Attention(dim, heads=heads, dim_head=dim_head, dropout=dropout) ff = FeedForward(dim, mlp_dim, dropout=dropout) self.layers.append( nn.ModuleList( [ PreNorm(dim, sa, norm=norm, norm_cond_dim=norm_cond_dim), PreNorm(dim, ff, norm=norm, norm_cond_dim=norm_cond_dim), ] ) ) def forward(self, x: torch.Tensor, *args): for attn, ff in self.layers: x = attn(x, *args) + x x = ff(x, *args) + x return x class AdaptiveLayerNorm1D(torch.nn.Module): def __init__(self, data_dim: int, norm_cond_dim: int): super().__init__() if data_dim <= 0: raise ValueError(f"data_dim must be positive, but got {data_dim}") if norm_cond_dim <= 0: raise ValueError(f"norm_cond_dim must be positive, but got {norm_cond_dim}") self.norm = torch.nn.LayerNorm( data_dim ) self.linear = torch.nn.Linear(norm_cond_dim, 2 * data_dim) torch.nn.init.zeros_(self.linear.weight) torch.nn.init.zeros_(self.linear.bias) def forward(self, x: torch.Tensor, t: torch.Tensor) -> torch.Tensor: x = self.norm(x) alpha, beta = self.linear(t).chunk(2, dim=-1) if x.dim() > 2: alpha = alpha.view(alpha.shape[0], *([1] * (x.dim() - 2)), alpha.shape[1]) beta = beta.view(beta.shape[0], *([1] * (x.dim() - 2)), beta.shape[1]) return x * (1 + alpha) + beta def normalization_layer(norm: Optional[str], dim: int, norm_cond_dim: int = -1): if norm == "batch": return torch.nn.BatchNorm1d(dim) elif norm == "layer": return torch.nn.LayerNorm(dim) elif norm == "ada": assert norm_cond_dim > 0, f"norm_cond_dim must be positive, got {norm_cond_dim}" return AdaptiveLayerNorm1D(dim, norm_cond_dim) elif norm is None: return torch.nn.Identity() else: raise ValueError(f"Unknown norm: {norm}") class PreNorm(nn.Module): def __init__(self, dim: int, fn: Callable, norm: str = "layer", norm_cond_dim: int = -1): super().__init__() self.norm = normalization_layer(norm, dim, norm_cond_dim) self.fn = fn def forward(self, x: torch.Tensor, *args, **kwargs): if isinstance(self.norm, AdaptiveLayerNorm1D): return self.fn(self.norm(x, *args), **kwargs) else: return self.fn(self.norm(x), **kwargs) # This class is from HaMeR (https://github.com/geopavlakos/hamer). class TransformerCrossAttn(nn.Module): def __init__( self, dim: int, depth: int, heads: int, dim_head: int, mlp_dim: int, dropout: float = 0.0, norm: str = "layer", norm_cond_dim: int = -1, context_dim: Optional[int] = None, ): super().__init__() self.layers = nn.ModuleList([]) for _ in range(depth): sa = Attention(dim, heads=heads, dim_head=dim_head, dropout=dropout) ca = CrossAttention( dim, context_dim=context_dim, heads=heads, dim_head=dim_head, dropout=dropout ) ff = FeedForward(dim, mlp_dim, dropout=dropout) self.layers.append( nn.ModuleList( [ PreNorm(dim, sa, norm=norm, norm_cond_dim=norm_cond_dim), PreNorm(dim, ca, norm=norm, norm_cond_dim=norm_cond_dim), PreNorm(dim, ff, norm=norm, norm_cond_dim=norm_cond_dim), ] ) ) def forward(self, x: torch.Tensor, *args, context=None, context_list=None): if context_list is None: context_list = [context] * len(self.layers) if len(context_list) != len(self.layers): raise ValueError(f"len(context_list) != len(self.layers) ({len(context_list)} != {len(self.layers)})") for i, (self_attn, cross_attn, ff) in enumerate(self.layers): x = self_attn(x, *args) + x x = cross_attn(x, *args, context=context_list[i]) + x x = ff(x, *args) + x return x # This class is from HaMeR (https://github.com/geopavlakos/hamer). class DropTokenDropout(nn.Module): def __init__(self, p: float = 0.1): super().__init__() if p < 0 or p > 1: raise ValueError( "dropout probability has to be between 0 and 1, " "but got {}".format(p) ) self.p = p def forward(self, x: torch.Tensor): if self.training and self.p > 0: zero_mask = torch.full_like(x[0, :, 0], self.p).bernoulli().bool() if zero_mask.any(): x = x[:, ~zero_mask, :] return x # This class is from HaMeR (https://github.com/geopavlakos/hamer). class ZeroTokenDropout(nn.Module): def __init__(self, p: float = 0.1): super().__init__() if p < 0 or p > 1: raise ValueError( "dropout probability has to be between 0 and 1, " "but got {}".format(p) ) self.p = p def forward(self, x: torch.Tensor): # x: (batch_size, seq_len, dim) if self.training and self.p > 0: zero_mask = torch.full_like(x[:, :, 0], self.p).bernoulli().bool() # Zero-out the masked tokens x[zero_mask, :] = 0 return x # This class is from HaMeR (https://github.com/geopavlakos/hamer). class TransformerDecoder(nn.Module): def __init__( self, num_tokens: int, token_dim: int, dim: int, depth: int, heads: int, mlp_dim: int, dim_head: int = 64, dropout: float = 0.0, emb_dropout: float = 0.0, emb_dropout_type: str = 'drop', norm: str = "layer", norm_cond_dim: int = -1, context_dim: Optional[int] = None, skip_token_embedding: bool = False, ): super().__init__() if not skip_token_embedding: self.to_token_embedding = nn.Linear(token_dim, dim) else: self.to_token_embedding = nn.Identity() if token_dim != dim: raise ValueError( f"token_dim ({token_dim}) != dim ({dim}) when skip_token_embedding is True" ) self.pos_embedding = nn.Parameter(torch.randn(1, num_tokens, dim)) if emb_dropout_type == "drop": self.dropout = DropTokenDropout(emb_dropout) elif emb_dropout_type == "zero": self.dropout = ZeroTokenDropout(emb_dropout) elif emb_dropout_type == "normal": self.dropout = nn.Dropout(emb_dropout) self.transformer = TransformerCrossAttn( dim, depth, heads, dim_head, mlp_dim, dropout, norm=norm, norm_cond_dim=norm_cond_dim, context_dim=context_dim, ) def forward(self, inp: torch.Tensor, *args, context=None, context_list=None): x = self.to_token_embedding(inp) b, n, _ = x.shape x = self.dropout(x) x = x + self.pos_embedding[:, :n] x = self.transformer(x, *args, context=context, context_list=context_list) return x def rot6d_to_rotmat(x: torch.Tensor) -> torch.Tensor: x = x.reshape(-1,2,3).permute(0, 2, 1).contiguous() a1 = x[:, :, 0] a2 = x[:, :, 1] b1 = F.normalize(a1) b2 = F.normalize(a2 - torch.einsum('bi,bi->b', b1, a2).unsqueeze(-1) * b1) b3 = torch.cross(b1, b2) return torch.stack((b1, b2, b3), dim=-1) def aa_to_rotmat(theta: torch.Tensor): norm = torch.norm(theta + 1e-8, p = 2, dim = 1) angle = torch.unsqueeze(norm, -1) normalized = torch.div(theta, angle) angle = angle * 0.5 v_cos = torch.cos(angle) v_sin = torch.sin(angle) quat = torch.cat([v_cos, v_sin * normalized], dim = 1) return quat_to_rotmat(quat) # This module is modified from MANOTransformerDecoderHead of HaMeR (https://github.com/geopavlakos/hamer). All cfg are directly initialized. class ContactTransformerDecoderHead(nn.Module): """ Cross-attention based MANO Transformer decoder """ def __init__(self): super().__init__() transformer_args = dict( num_tokens=1, token_dim=1, dim=1024, ) if cfg.MODEL.backbone_type in ['resnet-50', 'resnet-101', 'resnet-152', 'hrnet-w32', 'hrnet-w48']: context_dim = 2048 elif cfg.MODEL.backbone_type in ['vit-h-14']: context_dim = 1280 elif cfg.MODEL.backbone_type in ['vit-l-16']: context_dim = 1024 elif cfg.MODEL.backbone_type in ['vit-b-16']: context_dim = 768 elif cfg.MODEL.backbone_type in ['resnet-18', 'resnet-34']: context_dim = 512 elif cfg.MODEL.backbone_type in ['vit-s-16']: context_dim = 384 else: raise NotImplementedError TRANSFORMER_DECODER_CONFIG = {'depth': 6, 'heads': 8, 'mlp_dim': 1024, 'dim_head': 64, 'dropout': 0.0, 'emb_dropout': 0.0, 'norm': 'layer', 'context_dim': context_dim} transformer_args.update(dict(TRANSFORMER_DECODER_CONFIG)) self.transformer = TransformerDecoder( **transformer_args ) self.deccontact = nn.Linear(1024, 265) # 265 is number of foot vertices self.init_contact = nn.Parameter(torch.randn(1, 265, requires_grad=True)) def forward(self, x, **kwargs): batch_size = x.shape[0] device = x.device x = rearrange(x, 'b c h w -> b (h w) c') init_contact = self.init_contact.expand(batch_size, -1) pred_contact = init_contact token = torch.zeros(batch_size, 1, 1, device=x.device) token_out = self.transformer(token, context=x) token_out = token_out[:, 0] pred_contact = self.deccontact(token_out) + pred_contact pred_joint_contact = (torch.tensor(J_regressor_foot, dtype=torch.float32, device=device) @ pred_contact.T).T pred_joint_contact_openpose = (torch.tensor(J_regressor_foot_openpose, dtype=torch.float32, device=device) @ pred_contact.T).T pred_per_foot_contact = torch.logsumexp(pred_contact, dim=1) return pred_contact, pred_joint_contact, pred_joint_contact_openpose, pred_per_foot_contact