                    use_reentrant=False,
                )
                rgb_output = postprocess_rgb(rgb_out)
                final_output.update(rgb_output)

            if self.has_pose:
                pose = postprocess_pose(pose, self.pose_mode)
                final_output["camera_pose"] = pose  # B,7
                cross_out = checkpoint(
                    self.dpt_cross,
                    x_cross,
                    image_size=(img_info[0], img_info[1]),
                    use_reentrant=False,
                )
                tmp = postprocess(cross_out, self.depth_mode, self.conf_mode)
                final_output["pts3d_in_other_view"] = tmp.pop("pts3d")
                final_output["conf"] = tmp.pop("conf")
        return final_output

class DPTPts3dPoseSMPL(nn.Module):
    def __init__(self, net, has_conf=False, has_rgb=False, has_pose=False, has_msk=False):
        super(DPTPts3dPoseSMPL, self).__init__()
        self.return_all_layers = True  # backbone needs to return all layers
        self.depth_mode = net.depth_mode
        self.conf_mode = net.conf_mode
        self.pose_mode = net.pose_mode

        self.has_conf = has_conf
        self.has_rgb = has_rgb
        self.has_pose = has_pose
        self.has_msk = has_msk

        pts_channels = 3 + has_conf
        rgb_channels = has_rgb * 3
        feature_dim = 256
        last_dim = feature_dim // 2
        ed = net.enc_embed_dim
        dd = net.dec_embed_dim
        hooks_idx = [0, 1, 2, 3]
        dim_tokens = [ed, dd, dd, dd]
        head_type = "regression"
        output_width_ratio = 1

        pts_dpt_args = dict(
            output_width_ratio=output_width_ratio,
            num_channels=pts_channels,
            feature_dim=feature_dim,
            last_dim=last_dim,
            dim_tokens=dim_tokens,
            hooks_idx=hooks_idx,
            head_type=head_type,
        )
        rgb_dpt_args = dict(
            output_width_ratio=output_width_ratio,
            num_channels=rgb_channels,
            feature_dim=feature_dim,
            last_dim=last_dim,
            dim_tokens=dim_tokens,
            hooks_idx=hooks_idx,
            head_type=head_type,
        )
        if hooks_idx is not None:
            pts_dpt_args.update(hooks=hooks_idx)
            rgb_dpt_args.update(hooks=hooks_idx)

        self.dpt_self = DPTOutputAdapter_fix(**pts_dpt_args)
        dpt_init_args = {} if dim_tokens is None else {"dim_tokens_enc": dim_tokens}
        self.dpt_self.init(**dpt_init_args)

        self.final_transform = nn.ModuleList(
            [
                ConditionModulationBlock(
                    net.dec_embed_dim,
                    net.dec_num_heads,
                    mlp_ratio=4.0,
                    qkv_bias=True,
                    rope=net.rope,
                )
                for _ in range(2)
            ]
        )

        self.dpt_cross = DPTOutputAdapter_fix(**pts_dpt_args)
        dpt_init_args = {} if dim_tokens is None else {"dim_tokens_enc": dim_tokens}
        self.dpt_cross.init(**dpt_init_args)

        if has_rgb:
            self.dpt_rgb = DPTOutputAdapter_fix(**rgb_dpt_args)
            dpt_init_args = {} if dim_tokens is None else {"dim_tokens_enc": dim_tokens}
            self.dpt_rgb.init(**dpt_init_args)

        if has_pose:
            in_dim = net.dec_embed_dim
            self.pose_head = PoseDecoder(hidden_size=in_dim)
            self.in_dim = in_dim

        # MHMR Heads - Detection
        backbone_dim = net.backbone_dim
        self.bb_patch_size = net.bb_patch_size
        self.mlp_classif = regression_mlp([backbone_dim, backbone_dim, 1]) # bg or human
        self.mlp_offset = regression_mlp([backbone_dim, backbone_dim, 2]) # offset
        if has_msk:
            self.mlp_msk = SMPLDecoder(
                hidden_size=backbone_dim, target_dim=self.bb_patch_size**2, num_layers=2, mlp_ratio=1)

        # feature fuse
        self.mlp_fuse = SMPLDecoder(hidden_size=ed+backbone_dim, target_dim=dd, num_layers=2, mlp_ratio=4)

        # SMPL
        self.joint_rep_type, self.joint_rep_dim = '6d', 6
        self.nrot = 53
        self.num_body_joints = self.nrot - 1

        npose = self.joint_rep_dim * (self.num_body_joints + 1)
        self.npose = npose

        self.input_is_mean_shape = True
        self.num_betas = 10
        assert self.num_betas in [10, 11]

        # SMPL param heads
        self.deccam = SMPLDecoder(
                        hidden_size=in_dim,
                        target_dim=3,
                        num_layers=2,
                        mlp_ratio=4)
        self.decpose, self.decshape, self.decexpression = [
            SMPLDecoder(
                hidden_size=in_dim+backbone_dim,
                target_dim=od,
                num_layers=2,
                mlp_ratio=4) for od in [self.npose, self.num_betas, 10]]

        self.set_smpl_init()

    def set_smpl_init(self):
        """ Fetch saved SMPL parameters and register buffers."""
        mean_params = np.load(MEAN_PARAMS)
        if self.nrot == 53:
            init_body_pose = torch.eye(3).reshape(1,3,3).repeat(self.nrot,1,1)[:,:,:2].flatten(1).reshape(1, -1)
            init_body_pose[:,:24*6] = torch.from_numpy(mean_params['pose'][:]).float() # global_orient+body_pose from SMPL
        else:
            init_body_pose = torch.from_numpy(mean_params['pose'].astype(np.float32)).unsqueeze(0)

        init_betas = torch.from_numpy(mean_params['shape'].astype('float32')).unsqueeze(0)
        init_cam = torch.from_numpy(mean_params['cam'].astype(np.float32)).unsqueeze(0)
        init_betas_kid = torch.cat([init_betas, torch.zeros_like(init_betas[:,[0]])],1)
        init_expression = 0. * torch.from_numpy(mean_params['shape'].astype('float32')).unsqueeze(0)

        if self.num_betas == 11:
            init_betas = torch.cat([init_betas, torch.zeros_like(init_betas[:,:1])], 1)

        self.register_buffer('init_body_pose', init_body_pose)
        self.register_buffer('init_betas', init_betas)
        self.register_buffer('init_betas_kid', init_betas_kid)
        self.register_buffer('init_cam', init_cam)
        self.register_buffer('init_expression', init_expression)

    def detect_mhmr(self, x):
        with torch.cuda.amp.autocast(enabled=False):
            scores = postprocess_score(self.mlp_classif(x)) # per token detection score.
        return scores

    def segment(self, x):
        with torch.cuda.amp.autocast(enabled=False):
            msks = postprocess_score(self.mlp_msk(x))
        return msks

    def forward(self, x, img_info, **kwargs):
        if self.has_pose:
            pose_token = x[-1][:, 0].clone()
            n_humans_i = kwargs.get("n_humans")
            token = x[-1][:, 1:]

            with torch.cuda.amp.autocast(enabled=False):
                pose = self.pose_head(pose_token)
                if n_humans_i > 0:
                    smpl_token = kwargs.get("smpl_token")   # CUT3R smpl token (bs, 10, 768)
                    pred_body_pose = self.decpose(smpl_token) + self.init_body_pose
                    pred_betas = self.decshape(smpl_token) + self.init_betas
                    pred_cam = self.deccam(smpl_token[..., :self.in_dim])
                    pred_expression = self.decexpression(smpl_token) + self.init_expression
                    pred_smpl = [pred_body_pose, pred_betas, pred_cam, pred_expression]

            token_cross = token.clone()
            for blk in self.final_transform:
                token_cross = blk(token_cross, pose_token, kwargs.get("pos"))
            x = x[:-1] + [token]
            x_cross = x[:-1] + [token_cross]

        with torch.cuda.amp.autocast(enabled=False):
            self_out = checkpoint(
                self.dpt_self,
                x,
                image_size=(img_info[0], img_info[1]),
                # ret_feat=True,
                use_reentrant=False,
            )

            final_output = postprocess(self_out, self.depth_mode, self.conf_mode)
            final_output["pts3d_in_self_view"] = final_output.pop("pts3d")
            final_output["conf_self"] = final_output.pop("conf")

            if self.has_rgb:
                rgb_out = checkpoint(
                    self.dpt_rgb,
                    x,
                    image_size=(img_info[0], img_info[1]),
                    use_reentrant=False,
                )
                rgb_output = postprocess_rgb(rgb_out)
                final_output.update(rgb_output)

            if self.has_pose:
                pose = postprocess_pose(pose, self.pose_mode)
                final_output["camera_pose"] = pose  # B,7
                cross_out = checkpoint(
                    self.dpt_cross,
                    x_cross,
                    image_size=(img_info[0], img_info[1]),
                    use_reentrant=False,
                )
                tmp = postprocess(cross_out, self.depth_mode, self.conf_mode)
                final_output["pts3d_in_other_view"] = tmp.pop("pts3d")
                final_output["conf"] = tmp.pop("conf")

            if n_humans_i > 0:
                smpl_out = postprocess_smpl(pred_smpl, self.depth_mode)
                final_output.update(smpl_out)

        return final_output

class NaiveDPTPts3dPoseSMPL(nn.Module):
    def __init__(self, net, has_conf=False, has_rgb=False, has_pose=False):
        super(NaiveDPTPts3dPoseSMPL, self).__init__()
        self.return_all_layers = True  # backbone needs to return all layers
        self.depth_mode = net.depth_mode
        self.conf_mode = net.conf_mode
        self.pose_mode = net.pose_mode

        self.has_conf = has_conf
        self.has_rgb = has_rgb
        self.has_pose = has_pose

        pts_channels = 3 + has_conf
        rgb_channels = has_rgb * 3
        feature_dim = 256
        last_dim = feature_dim // 2
        ed = net.enc_embed_dim
        dd = net.dec_embed_dim


            if self.has_pose:
                pose = postprocess_pose(pose, self.pose_mode)
                final_output["camera_pose"] = pose  # B,7
                cross_out = checkpoint(
                    self.dpt_cross,
                    x_cross,
                    image_size=(img_info[0], img_info[1]),
                    use_reentrant=False,
                )
                tmp = postprocess(cross_out, self.depth_mode, self.conf_mode)
                final_output["pts3d_in_other_view"] = tmp.pop("pts3d")
                final_output["conf"] = tmp.pop("conf")

            if n_humans_i > 0:
                smpl_out = postprocess_smpl(pred_smpl, self.depth_mode)
                final_output.update(smpl_out)

        return final_output

class NaiveDPTPts3dPoseSMPL(nn.Module):
    def __init__(self, net, has_conf=False, has_rgb=False, has_pose=False):
        super(NaiveDPTPts3dPoseSMPL, self).__init__()
        self.return_all_layers = True  # backbone needs to return all layers
        self.depth_mode = net.depth_mode
        self.conf_mode = net.conf_mode
        self.pose_mode = net.pose_mode

        self.has_conf = has_conf
        self.has_rgb = has_rgb
        self.has_pose = has_pose

        pts_channels = 3 + has_conf
        rgb_channels = has_rgb * 3
        feature_dim = 256
        last_dim = feature_dim // 2
        ed = net.enc_embed_dim
        dd = net.dec_embed_dim
        hooks_idx = [0, 1, 2, 3]
        dim_tokens = [ed, dd, dd, dd]
        head_type = "regression"
        output_width_ratio = 1

        pts_dpt_args = dict(
            output_width_ratio=output_width_ratio,
            num_channels=pts_channels,
            feature_dim=feature_dim,
            last_dim=last_dim,
            dim_tokens=dim_tokens,
            hooks_idx=hooks_idx,
            head_type=head_type,
        )
        rgb_dpt_args = dict(
            output_width_ratio=output_width_ratio,
            num_channels=rgb_channels,
            feature_dim=feature_dim,
            last_dim=last_dim,
            dim_tokens=dim_tokens,
            hooks_idx=hooks_idx,
            head_type=head_type,
        )
        if hooks_idx is not None:
            pts_dpt_args.update(hooks=hooks_idx)
            rgb_dpt_args.update(hooks=hooks_idx)

        self.dpt_self = DPTOutputAdapter_fix(**pts_dpt_args)
        dpt_init_args = {} if dim_tokens is None else {"dim_tokens_enc": dim_tokens}
        self.dpt_self.init(**dpt_init_args)

        self.final_transform = nn.ModuleList(
            [
                ConditionModulationBlock(
                    net.dec_embed_dim,
                    net.dec_num_heads,
                    mlp_ratio=4.0,
                    qkv_bias=True,
                    rope=net.rope,
                )
                for _ in range(2)
            ]
        )

        self.dpt_cross = DPTOutputAdapter_fix(**pts_dpt_args)
        dpt_init_args = {} if dim_tokens is None else {"dim_tokens_enc": dim_tokens}
        self.dpt_cross.init(**dpt_init_args)

        if has_rgb:
            self.dpt_rgb = DPTOutputAdapter_fix(**rgb_dpt_args)
            dpt_init_args = {} if dim_tokens is None else {"dim_tokens_enc": dim_tokens}
            self.dpt_rgb.init(**dpt_init_args)

        if has_pose:
            in_dim = net.dec_embed_dim
            self.pose_head = PoseDecoder(hidden_size=in_dim)

        # MHMR Heads - Detection
        backbone_dim = net.backbone_dim
        self.bb_patch_size = net.bb_patch_size
        self.mlp_classif = regression_mlp([backbone_dim, backbone_dim, 1]) # bg or human
        self.mlp_offset = regression_mlp([backbone_dim, backbone_dim, 2]) # offset

        # SMPL
        self.joint_rep_type, self.joint_rep_dim = '6d', 6
        self.nrot = 53
        self.num_body_joints = self.nrot - 1

        npose = self.joint_rep_dim * (self.num_body_joints + 1)
        self.npose = npose

        self.input_is_mean_shape = True
        self.num_betas = 10
        assert self.num_betas in [10, 11]

        # MHMR Heads - SMPL
        self.decpose, self.decshape, self.deccam, self.decexpression = [
            nn.Linear(1024, od) for od in [self.npose, self.num_betas, 3, 10]] # MLP(1024, x)

        self.set_smpl_init()

    def set_smpl_init(self):
        """ Fetch saved SMPL parameters and register buffers."""
        mean_params = np.load(MEAN_PARAMS)
        if self.nrot == 53:
            init_body_pose = torch.eye(3).reshape(1,3,3).repeat(self.nrot,1,1)[:,:,:2].flatten(1).reshape(1, -1)
            init_body_pose[:,:24*6] = torch.from_numpy(mean_params['pose'][:]).float() # global_orient+body_pose from SMPL
        else:
            init_body_pose = torch.from_numpy(mean_params['pose'].astype(np.float32)).unsqueeze(0)

        init_betas = torch.from_numpy(mean_params['shape'].astype('float32')).unsqueeze(0)
        init_cam = torch.from_numpy(mean_params['cam'].astype(np.float32)).unsqueeze(0)
        init_betas_kid = torch.cat([init_betas, torch.zeros_like(init_betas[:,[0]])],1)
        init_expression = 0. * torch.from_numpy(mean_params['shape'].astype('float32')).unsqueeze(0)

        if self.num_betas == 11:
            init_betas = torch.cat([init_betas, torch.zeros_like(init_betas[:,:1])], 1)

        self.register_buffer('init_body_pose', init_body_pose)
        self.register_buffer('init_betas', init_betas)
        self.register_buffer('init_betas_kid', init_betas_kid)
        self.register_buffer('init_cam', init_cam)
        self.register_buffer('init_expression', init_expression)

    def detect_mhmr(self, x):
        with torch.cuda.amp.autocast(enabled=False):
            scores = postprocess_score(self.mlp_classif(x)) # per token detection score.
        return scores

    def forward(self, x, img_info, **kwargs):
        if self.has_pose:
            pose_token = x[-1][:, 0].clone()
            n_humans_i = kwargs.get("n_humans")
            token = x[-1][:, 1:]

            with torch.cuda.amp.autocast(enabled=False):
                pose = self.pose_head(pose_token)
                if n_humans_i > 0:
                    smpl_token = kwargs.get("smpl_token")
                    decoders = [self.decpose, self.decshape, self.deccam, self.decexpression]
                    inits = [self.init_body_pose, self.init_betas, self.init_cam, self.init_expression]
                    pred_smpl = [d(smpl_token) + i for d, i in zip(decoders, inits)]

            token_cross = token.clone()
            for blk in self.final_transform:
                token_cross = blk(token_cross, pose_token, kwargs.get("pos"))
            x = x[:-1] + [token]
            x_cross = x[:-1] + [token_cross]

        with torch.cuda.amp.autocast(enabled=False):
            self_out = checkpoint(
                self.dpt_self,
                x,
                image_size=(img_info[0], img_info[1]),
                use_reentrant=False,
            )

            final_output = postprocess(self_out, self.depth_mode, self.conf_mode)
            final_output["pts3d_in_self_view"] = final_output.pop("pts3d")
            final_output["conf_self"] = final_output.pop("conf")

            if self.has_rgb:
                rgb_out = checkpoint(
                    self.dpt_rgb,
                    x,
                    image_size=(img_info[0], img_info[1]),
                    use_reentrant=False,
                )
                rgb_output = postprocess_rgb(rgb_out)
                final_output.update(rgb_output)

            if self.has_pose:
                pose = postprocess_pose(pose, self.pose_mode)
                final_output["camera_pose"] = pose  # B,7
                cross_out = checkpoint(
                    self.dpt_cross,
                    x_cross,
                    image_size=(img_info[0], img_info[1]),
                    use_reentrant=False,
                )
                tmp = postprocess(cross_out, self.depth_mode, self.conf_mode)
                final_output["pts3d_in_other_view"] = tmp.pop("pts3d")
                final_output["conf"] = tmp.pop("conf")

            if n_humans_i > 0:
                smpl_out = postprocess_smpl(pred_smpl, self.depth_mode, naive_mode=True)
                final_output.update(smpl_out)

        return final_output


init

    postprocess_pose,
    reg_dense_conf,
    postprocess_smpl,
    postprocess_score,
)
import dust3r.utils.path_to_croco  # noqa: F401
from models.dpt_block import DPTOutputAdapter, Interpolate  # noqa
from dust3r.utils.camera import pose_encoding_to_camera, PoseDecoder
from dust3r.blocks import ConditionModulationBlock
from torch.utils.checkpoint import checkpoint
from dust3r.smpl_model import SMPLDecoder, MEAN_PARAMS, regression_mlp
import numpy as np

class DPTOutputAdapter_fix(DPTOutputAdapter):
    """
    Adapt croco's DPTOutputAdapter implementation for dust3r:
    remove duplicated weigths, and fix forward for dust3r
    """

    def init(self, dim_tokens_enc=768):
        super().init(dim_tokens_enc)

        del self.act_1_postprocess
        del self.act_2_postprocess
        del self.act_3_postprocess
        del self.act_4_postprocess

    def forward(self, encoder_tokens: List[torch.Tensor], image_size=None, ret_feat=False):
        assert (
            self.dim_tokens_enc is not None
        ), "Need to call init(dim_tokens_enc) function first"

        image_size = self.image_size if image_size is None else image_size
        H, W = image_size

        N_H = H // (self.stride_level * self.P_H)
        N_W = W // (self.stride_level * self.P_W)

        layers = [encoder_tokens[hook] for hook in self.hooks]

        layers = [self.adapt_tokens(l) for l in layers]

        layers = [
            rearrange(l, "b (nh nw) c -> b c nh nw", nh=N_H, nw=N_W) for l in layers
        ]


forward

        init_betas = torch.from_numpy(mean_params['shape'].astype('float32')).unsqueeze(0)
        init_cam = torch.from_numpy(mean_params['cam'].astype(np.float32)).unsqueeze(0)
        init_betas_kid = torch.cat([init_betas, torch.zeros_like(init_betas[:,[0]])],1)
        init_expression = 0. * torch.from_numpy(mean_params['shape'].astype('float32')).unsqueeze(0)

        if self.num_betas == 11:
            init_betas = torch.cat([init_betas, torch.zeros_like(init_betas[:,:1])], 1)

        self.register_buffer('init_body_pose', init_body_pose)
        self.register_buffer('init_betas', init_betas)
        self.register_buffer('init_betas_kid', init_betas_kid)
        self.register_buffer('init_cam', init_cam)
        self.register_buffer('init_expression', init_expression)

    def detect_mhmr(self, x):
        with torch.cuda.amp.autocast(enabled=False):
            scores = postprocess_score(self.mlp_classif(x)) # per token detection score.
        return scores

    def forward(self, x, img_info, **kwargs):
        if self.has_pose:
            pose_token = x[-1][:, 0].clone()
            n_humans_i = kwargs.get("n_humans")
            token = x[-1][:, 1:]

            with torch.cuda.amp.autocast(enabled=False):
                pose = self.pose_head(pose_token)
                if n_humans_i > 0:
                    smpl_token = kwargs.get("smpl_token")
                    decoders = [self.decpose, self.decshape, self.deccam, self.decexpression]
                    inits = [self.init_body_pose, self.init_betas, self.init_cam, self.init_expression]
                    pred_smpl = [d(smpl_token) + i for d, i in zip(decoders, inits)]

            token_cross = token.clone()
            for blk in self.final_transform:
                token_cross = blk(token_cross, pose_token, kwargs.get("pos"))
            x = x[:-1] + [token]
            x_cross = x[:-1] + [token_cross]

        with torch.cuda.amp.autocast(enabled=False):
            self_out = checkpoint(
                self.dpt_self,
                x,
                image_size=(img_info[0], img_info[1]),
                use_reentrant=False,
            )

            final_output = postprocess(self_out, self.depth_mode, self.conf_mode)
            final_output["pts3d_in_self_view"] = final_output.pop("pts3d")
            final_output["conf_self"] = final_output.pop("conf")

            if self.has_rgb:
                rgb_out = checkpoint(
                    self.dpt_rgb,
                    x,
                    image_size=(img_info[0], img_info[1]),
                    use_reentrant=False,
                )
                rgb_output = postprocess_rgb(rgb_out)
                final_output.update(rgb_output)

            if self.has_pose:
                pose = postprocess_pose(pose, self.pose_mode)
                final_output["camera_pose"] = pose  # B,7
                cross_out = checkpoint(
                    self.dpt_cross,
                    x_cross,
                    image_size=(img_info[0], img_info[1]),
                    use_reentrant=False,
                )
                tmp = postprocess(cross_out, self.depth_mode, self.conf_mode)
                final_output["pts3d_in_other_view"] = tmp.pop("pts3d")
                final_output["conf"] = tmp.pop("conf")

            if n_humans_i > 0:
                smpl_out = postprocess_smpl(pred_smpl, self.depth_mode, naive_mode=True)
                final_output.update(smpl_out)

        return final_output


                pose = postprocess_pose(pose, self.pose_mode)
                final_output["camera_pose"] = pose  # B,7
                cross_out = checkpoint(
                    self.dpt_cross,
                    x_cross,
                    image_size=(img_info[0], img_info[1]),
                    use_reentrant=False,
                )
                tmp = postprocess(cross_out, self.depth_mode, self.conf_mode)
                final_output["pts3d_in_other_view"] = tmp.pop("pts3d")
                final_output["conf"] = tmp.pop("conf")

            if n_humans_i > 0:
                smpl_out = postprocess_smpl(pred_smpl, self.depth_mode)
                final_output.update(smpl_out)

        return final_output

class NaiveDPTPts3dPoseSMPL(nn.Module):
    def __init__(self, net, has_conf=False, has_rgb=False, has_pose=False):
        super(NaiveDPTPts3dPoseSMPL, self).__init__()
        self.return_all_layers = True  # backbone needs to return all layers
        self.depth_mode = net.depth_mode
        self.conf_mode = net.conf_mode
        self.pose_mode = net.pose_mode

        self.has_conf = has_conf
        self.has_rgb = has_rgb
        self.has_pose = has_pose

        pts_channels = 3 + has_conf
        rgb_channels = has_rgb * 3
        feature_dim = 256
        last_dim = feature_dim // 2
        ed = net.enc_embed_dim
        dd = net.dec_embed_dim
        hooks_idx = [0, 1, 2, 3]
        dim_tokens = [ed, dd, dd, dd]
        head_type = "regression"
        output_width_ratio = 1

        pts_dpt_args = dict(
            output_width_ratio=output_width_ratio,
            num_channels=pts_channels,
            feature_dim=feature_dim,
            last_dim=last_dim,
            dim_tokens=dim_tokens,
            hooks_idx=hooks_idx,
            head_type=head_type,
        )
        rgb_dpt_args = dict(
            output_width_ratio=output_width_ratio,
            num_channels=rgb_channels,
            feature_dim=feature_dim,
            last_dim=last_dim,
            dim_tokens=dim_tokens,
            hooks_idx=hooks_idx,
            head_type=head_type,
        )
        if hooks_idx is not None:
            pts_dpt_args.update(hooks=hooks_idx)
            rgb_dpt_args.update(hooks=hooks_idx)

        self.dpt_self = DPTOutputAdapter_fix(**pts_dpt_args)
        dpt_init_args = {} if dim_tokens is None else {"dim_tokens_enc": dim_tokens}
        self.dpt_self.init(**dpt_init_args)

        self.final_transform = nn.ModuleList(
            [
                ConditionModulationBlock(
                    net.dec_embed_dim,
                    net.dec_num_heads,
                    mlp_ratio=4.0,
                    qkv_bias=True,
                    rope=net.rope,
                )
                for _ in range(2)
            ]
        )

        self.dpt_cross = DPTOutputAdapter_fix(**pts_dpt_args)
        dpt_init_args = {} if dim_tokens is None else {"dim_tokens_enc": dim_tokens}
        self.dpt_cross.init(**dpt_init_args)

        if has_rgb:
            self.dpt_rgb = DPTOutputAdapter_fix(**rgb_dpt_args)
            dpt_init_args = {} if dim_tokens is None else {"dim_tokens_enc": dim_tokens}
            self.dpt_rgb.init(**dpt_init_args)

        if has_pose:
            in_dim = net.dec_embed_dim
            self.pose_head = PoseDecoder(hidden_size=in_dim)

        # MHMR Heads - Detection
        backbone_dim = net.backbone_dim
        self.bb_patch_size = net.bb_patch_size
        self.mlp_classif = regression_mlp([backbone_dim, backbone_dim, 1]) # bg or human
        self.mlp_offset = regression_mlp([backbone_dim, backbone_dim, 2]) # offset

        # SMPL
        self.joint_rep_type, self.joint_rep_dim = '6d', 6
        self.nrot = 53
        self.num_body_joints = self.nrot - 1

        npose = self.joint_rep_dim * (self.num_body_joints + 1)
        self.npose = npose

        self.input_is_mean_shape = True
        self.num_betas = 10
        assert self.num_betas in [10, 11]

        # MHMR Heads - SMPL
        self.decpose, self.decshape, self.deccam, self.decexpression = [
            nn.Linear(1024, od) for od in [self.npose, self.num_betas, 3, 10]] # MLP(1024, x)

        self.set_smpl_init()

    def set_smpl_init(self):
        """ Fetch saved SMPL parameters and register buffers."""
        mean_params = np.load(MEAN_PARAMS)
        if self.nrot == 53:
            init_body_pose = torch.eye(3).reshape(1,3,3).repeat(self.nrot,1,1)[:,:,:2].flatten(1).reshape(1, -1)
            init_body_pose[:,:24*6] = torch.from_numpy(mean_params['pose'][:]).float() # global_orient+body_pose from SMPL
        else:
            init_body_pose = torch.from_numpy(mean_params['pose'].astype(np.float32)).unsqueeze(0)

        init_betas = torch.from_numpy(mean_params['shape'].astype('float32')).unsqueeze(0)
        init_cam = torch.from_numpy(mean_params['cam'].astype(np.float32)).unsqueeze(0)
        init_betas_kid = torch.cat([init_betas, torch.zeros_like(init_betas[:,[0]])],1)
        init_expression = 0. * torch.from_numpy(mean_params['shape'].astype('float32')).unsqueeze(0)

        if self.num_betas == 11:
            init_betas = torch.cat([init_betas, torch.zeros_like(init_betas[:,:1])], 1)

        self.register_buffer('init_body_pose', init_body_pose)
        self.register_buffer('init_betas', init_betas)


        self.mlp_offset = regression_mlp([backbone_dim, backbone_dim, 2]) # offset

        # SMPL
        self.joint_rep_type, self.joint_rep_dim = '6d', 6
        self.nrot = 53
        self.num_body_joints = self.nrot - 1

        npose = self.joint_rep_dim * (self.num_body_joints + 1)
        self.npose = npose

        self.input_is_mean_shape = True
        self.num_betas = 10
        assert self.num_betas in [10, 11]

        # MHMR Heads - SMPL
        self.decpose, self.decshape, self.deccam, self.decexpression = [
            nn.Linear(1024, od) for od in [self.npose, self.num_betas, 3, 10]] # MLP(1024, x)

        self.set_smpl_init()

    def set_smpl_init(self):
        """ Fetch saved SMPL parameters and register buffers."""
        mean_params = np.load(MEAN_PARAMS)
        if self.nrot == 53:
            init_body_pose = torch.eye(3).reshape(1,3,3).repeat(self.nrot,1,1)[:,:,:2].flatten(1).reshape(1, -1)
            init_body_pose[:,:24*6] = torch.from_numpy(mean_params['pose'][:]).float() # global_orient+body_pose from SMPL
        else:
            init_body_pose = torch.from_numpy(mean_params['pose'].astype(np.float32)).unsqueeze(0)

        init_betas = torch.from_numpy(mean_params['shape'].astype('float32')).unsqueeze(0)
        init_cam = torch.from_numpy(mean_params['cam'].astype(np.float32)).unsqueeze(0)
        init_betas_kid = torch.cat([init_betas, torch.zeros_like(init_betas[:,[0]])],1)
        init_expression = 0. * torch.from_numpy(mean_params['shape'].astype('float32')).unsqueeze(0)

        if self.num_betas == 11:
            init_betas = torch.cat([init_betas, torch.zeros_like(init_betas[:,:1])], 1)

        self.register_buffer('init_body_pose', init_body_pose)
        self.register_buffer('init_betas', init_betas)
        self.register_buffer('init_betas_kid', init_betas_kid)
        self.register_buffer('init_cam', init_cam)
        self.register_buffer('init_expression', init_expression)

    def detect_mhmr(self, x):
        with torch.cuda.amp.autocast(enabled=False):
            scores = postprocess_score(self.mlp_classif(x)) # per token detection score.
        return scores

    def forward(self, x, img_info, **kwargs):
        if self.has_pose:
            pose_token = x[-1][:, 0].clone()
            n_humans_i = kwargs.get("n_humans")
            token = x[-1][:, 1:]

            with torch.cuda.amp.autocast(enabled=False):
                pose = self.pose_head(pose_token)
                if n_humans_i > 0:
                    smpl_token = kwargs.get("smpl_token")
                    decoders = [self.decpose, self.decshape, self.deccam, self.decexpression]
                    inits = [self.init_body_pose, self.init_betas, self.init_cam, self.init_expression]
                    pred_smpl = [d(smpl_token) + i for d, i in zip(decoders, inits)]


        if self.nrot == 53:
            init_body_pose = torch.eye(3).reshape(1,3,3).repeat(self.nrot,1,1)[:,:,:2].flatten(1).reshape(1, -1)
            init_body_pose[:,:24*6] = torch.from_numpy(mean_params['pose'][:]).float() # global_orient+body_pose from SMPL
        else:
            init_body_pose = torch.from_numpy(mean_params['pose'].astype(np.float32)).unsqueeze(0)

        init_betas = torch.from_numpy(mean_params['shape'].astype('float32')).unsqueeze(0)
        init_cam = torch.from_numpy(mean_params['cam'].astype(np.float32)).unsqueeze(0)
        init_betas_kid = torch.cat([init_betas, torch.zeros_like(init_betas[:,[0]])],1)
        init_expression = 0. * torch.from_numpy(mean_params['shape'].astype('float32')).unsqueeze(0)

        if self.num_betas == 11:
            init_betas = torch.cat([init_betas, torch.zeros_like(init_betas[:,:1])], 1)

        self.register_buffer('init_body_pose', init_body_pose)
        self.register_buffer('init_betas', init_betas)
        self.register_buffer('init_betas_kid', init_betas_kid)
        self.register_buffer('init_cam', init_cam)
        self.register_buffer('init_expression', init_expression)

    def detect_mhmr(self, x):
        with torch.cuda.amp.autocast(enabled=False):
            scores = postprocess_score(self.mlp_classif(x)) # per token detection score.
        return scores

    def forward(self, x, img_info, **kwargs):
        if self.has_pose:
            pose_token = x[-1][:, 0].clone()
            n_humans_i = kwargs.get("n_humans")
            token = x[-1][:, 1:]

            with torch.cuda.amp.autocast(enabled=False):
                pose = self.pose_head(pose_token)
                if n_humans_i > 0:
                    smpl_token = kwargs.get("smpl_token")
                    decoders = [self.decpose, self.decshape, self.deccam, self.decexpression]
                    inits = [self.init_body_pose, self.init_betas, self.init_cam, self.init_expression]
                    pred_smpl = [d(smpl_token) + i for d, i in zip(decoders, inits)]

            token_cross = token.clone()
            for blk in self.final_transform:
                token_cross = blk(token_cross, pose_token, kwargs.get("pos"))
            x = x[:-1] + [token]
            x_cross = x[:-1] + [token_cross]


segment

        init_betas = torch.from_numpy(mean_params['shape'].astype('float32')).unsqueeze(0)
        init_cam = torch.from_numpy(mean_params['cam'].astype(np.float32)).unsqueeze(0)
        init_betas_kid = torch.cat([init_betas, torch.zeros_like(init_betas[:,[0]])],1)
        init_expression = 0. * torch.from_numpy(mean_params['shape'].astype('float32')).unsqueeze(0)

        if self.num_betas == 11:
            init_betas = torch.cat([init_betas, torch.zeros_like(init_betas[:,:1])], 1)

        self.register_buffer('init_body_pose', init_body_pose)
        self.register_buffer('init_betas', init_betas)
        self.register_buffer('init_betas_kid', init_betas_kid)
        self.register_buffer('init_cam', init_cam)
        self.register_buffer('init_expression', init_expression)

    def detect_mhmr(self, x):
        with torch.cuda.amp.autocast(enabled=False):
            scores = postprocess_score(self.mlp_classif(x)) # per token detection score.
        return scores

    def segment(self, x):
        with torch.cuda.amp.autocast(enabled=False):
            msks = postprocess_score(self.mlp_msk(x))
        return msks

    def forward(self, x, img_info, **kwargs):
        if self.has_pose:
            pose_token = x[-1][:, 0].clone()
            n_humans_i = kwargs.get("n_humans")
            token = x[-1][:, 1:]

            with torch.cuda.amp.autocast(enabled=False):
                pose = self.pose_head(pose_token)
                if n_humans_i > 0:
                    smpl_token = kwargs.get("smpl_token")   # CUT3R smpl token (bs, 10, 768)
                    pred_body_pose = self.decpose(smpl_token) + self.init_body_pose
                    pred_betas = self.decshape(smpl_token) + self.init_betas
                    pred_cam = self.deccam(smpl_token[..., :self.in_dim])
                    pred_expression = self.decexpression(smpl_token) + self.init_expression
                    pred_smpl = [pred_body_pose, pred_betas, pred_cam, pred_expression]

            token_cross = token.clone()
            for blk in self.final_transform:
                token_cross = blk(token_cross, pose_token, kwargs.get("pos"))


# Copyright (C) 2024-present Naver Corporation. All rights reserved.
# Licensed under CC BY-NC-SA 4.0 (non-commercial use only).
#
# --------------------------------------------------------
# modified from DUSt3R

import torch
import torch.nn.functional as F
import roma

def postprocess(out, depth_mode, conf_mode, pos_z=False):
    """
    extract 3D points/confidence from prediction head output
    """
    fmap = out.permute(0, 2, 3, 1)  # B,H,W,3
    res = dict(pts3d=reg_dense_depth(fmap[:, :, :, 0:3], mode=depth_mode, pos_z=pos_z))

    if conf_mode is not None:
        res["conf"] = reg_dense_conf(fmap[:, :, :, 3], mode=conf_mode)
    return res

def postprocess_rgb(out, eps=1e-6):
    fmap = out.permute(0, 2, 3, 1)  # B,H,W,3
    res = torch.sigmoid(fmap) * (1 - 2 * eps) + eps
    res = (res - 0.5) * 2
    return dict(rgb=res)

def postprocess_score(scores, eps=1e-6):
    out = torch.sigmoid(scores) * (1 - 2 * eps) + eps
    return out

def postprocess_pose(out, mode, inverse=False):
    """
    extract pose from prediction head output
    """
    mode, vmin, vmax = mode

    no_bounds = (vmin == -float("inf")) and (vmax == float("inf"))
    assert no_bounds
    trans = out[..., 0:3]
    quats = out[..., 3:7]

    if mode == "linear":
        if no_bounds:
            return trans  # [-inf, +inf]
        return trans.clip(min=vmin, max=vmax)

    d = trans.norm(dim=-1, keepdim=True)

    if mode == "square":
        if inverse:
            scale = d / d.square().clip(min=1e-8)
        else:
            scale = d.square() / d.clip(min=1e-8)

    if mode == "exp":
        if inverse:
            scale = d / torch.expm1(d).clip(min=1e-8)
        else:
            scale = torch.expm1(d) / d.clip(min=1e-8)

    trans = trans * scale
    quats = standardize_quaternion(quats)

    return torch.cat([trans, quats], dim=-1)

def postprocess_smpl(out, mode, pos_z=False, naive_mode=False):
    """
    extract SMPL parameters from prediction head output
    """
    pred_body_pose, pred_betas, pred_cam, pred_expression = out
    bs, max_humans = pred_body_pose.shape[:2]

    res = dict(smpl_shape=pred_betas)
    if naive_mode:
        res['smpl_transl'] = pred_cam # MHMR
    else:
        res['smpl_transl'] = reg_dense_depth(pred_cam, mode=mode, pos_z=pos_z)

    res['smpl_rotmat'] = rot6d_to_rotmat(
        pred_body_pose.view(-1, pred_body_pose.shape[-1]),
        naive_mode=naive_mode,
        ).view(bs, max_humans, -1, 3, 3)
    res['smpl_expression'] = pred_expression

    return res

def postprocess_pose_conf(out):
    fmap = out.permute(0, 2, 3, 1)  # B,H,W,1
    return dict(pose_conf=torch.sigmoid(fmap))

def postprocess_desc(out, depth_mode, conf_mode, desc_dim, double_channel=False):
    """
    extract 3D points/confidence from prediction head output
    """
    fmap = out.permute(0, 2, 3, 1)  # B,H,W,3
    res = dict(pts3d=reg_dense_depth(fmap[:, :, :, 0:3], mode=depth_mode))

    if conf_mode is not None:
        res["conf"] = reg_dense_conf(fmap[:, :, :, 3], mode=conf_mode)

    if double_channel:
        res["pts3d_self"] = reg_dense_depth(
            fmap[
                :, :, :, 3 + int(conf_mode is not None) : 6 + int(conf_mode is not None)
            ],
            mode=depth_mode,
        )
        if conf_mode is not None:
            res["conf_self"] = reg_dense_conf(
                fmap[:, :, :, 6 + int(conf_mode is not None)], mode=conf_mode
            )

    start = (
        3
        + int(conf_mode is not None)
        + int(double_channel) * (3 + int(conf_mode is not None))
    )
    res["desc"] = reg_desc(fmap[:, :, :, start : start + desc_dim], mode="norm")
    res["desc_conf"] = reg_dense_conf(fmap[:, :, :, start + desc_dim], mode=conf_mode)
    assert start + desc_dim + 1 == fmap.shape[-1]

    return res

def reg_desc(desc, mode="norm"):
    if "norm" in mode:
        desc = desc / desc.norm(dim=-1, keepdim=True)
    else:
        raise ValueError(f"Unknown desc mode {mode}")
    return desc

def reg_dense_depth(xyz, mode, pos_z=False):
    """
    extract 3D points from prediction head output
    """
    mode, vmin, vmax = mode

    no_bounds = (vmin == -float("inf")) and (vmax == float("inf"))
    assert no_bounds

    if mode == "linear":
        if no_bounds:
            return xyz  # [-inf, +inf]
        return xyz.clip(min=vmin, max=vmax)

    if pos_z:
        sign = torch.sign(xyz[..., -1:])
        xyz *= sign
    d = xyz.norm(dim=-1, keepdim=True)
    xyz = xyz / d.clip(min=1e-8)

    if mode == "square":
        return xyz * d.square()

    if mode == "exp":
        return xyz * torch.expm1(d)

    raise ValueError(f"bad {mode=}")

def reg_dense_conf(x, mode):
    """
    extract confidence from prediction head output
    """
    mode, vmin, vmax = mode
    if mode == "exp":
        return vmin + x.exp().clip(max=vmax - vmin)
    if mode == "sigmoid":
        return (vmax - vmin) * torch.sigmoid(x) + vmin
    raise ValueError(f"bad {mode=}")

def standardize_quaternion(quaternions: torch.Tensor) -> torch.Tensor:
    """
    Convert a unit quaternion to a standard form: one in which the real
    part is non negative.

    Args:
        quaternions: Quaternions with real part first,
            as tensor of shape (..., 4).

    Returns:
        Standardized quaternions as tensor of shape (..., 4).
    """
    quaternions = F.normalize(quaternions, p=2, dim=-1)
    return torch.where(quaternions[..., 0:1] < 0, -quaternions, quaternions)

def rot6d_to_rotmat(x, naive_mode=False):
    """
    6D rotation representation to 3x3 rotation matrix.
    Args:
        x: (nvh,num_joints*6) Batch of 6-D rotation representations.
    Returns:
        torch.Tensor: Batch of corresponding rotation matrices with shape (nvh*num_joints,3,3).
    """
    if naive_mode:
        x = x.reshape(-1,2,3).permute(0, 2, 1).contiguous() # (nvh, num_joints*6) -> (nvh*num_joints, 3, 2) # inherited from MHMR
    else:
        x = x.reshape(-1,3,2).contiguous() # (nvh, num_joints*6) -> (nvh*num_joints, 3, 2)
    y = roma.special_gramschmidt(x, epsilon=1e-6) # (nvh*num_joints, 3, 3)
    return y


postprocess

# Copyright (C) 2024-present Naver Corporation. All rights reserved.
# Licensed under CC BY-NC-SA 4.0 (non-commercial use only).
#
# --------------------------------------------------------
# modified from DUSt3R

import torch
import torch.nn.functional as F
import roma

def postprocess(out, depth_mode, conf_mode, pos_z=False):
    """
    extract 3D points/confidence from prediction head output
    """
    fmap = out.permute(0, 2, 3, 1)  # B,H,W,3
    res = dict(pts3d=reg_dense_depth(fmap[:, :, :, 0:3], mode=depth_mode, pos_z=pos_z))

    if conf_mode is not None:
        res["conf"] = reg_dense_conf(fmap[:, :, :, 3], mode=conf_mode)
    return res

def postprocess_rgb(out, eps=1e-6):
    fmap = out.permute(0, 2, 3, 1)  # B,H,W,3
    res = torch.sigmoid(fmap) * (1 - 2 * eps) + eps
    res = (res - 0.5) * 2
    return dict(rgb=res)

def postprocess_score(scores, eps=1e-6):
    out = torch.sigmoid(scores) * (1 - 2 * eps) + eps
    return out

def postprocess_pose(out, mode, inverse=False):
    """
    extract pose from prediction head output
    """
    mode, vmin, vmax = mode


#
# --------------------------------------------------------
# modified from DUSt3R

import torch
import torch.nn.functional as F
import roma

def postprocess(out, depth_mode, conf_mode, pos_z=False):
    """
    extract 3D points/confidence from prediction head output
    """
    fmap = out.permute(0, 2, 3, 1)  # B,H,W,3
    res = dict(pts3d=reg_dense_depth(fmap[:, :, :, 0:3], mode=depth_mode, pos_z=pos_z))

    if conf_mode is not None:
        res["conf"] = reg_dense_conf(fmap[:, :, :, 3], mode=conf_mode)
    return res

def postprocess_rgb(out, eps=1e-6):
    fmap = out.permute(0, 2, 3, 1)  # B,H,W,3
    res = torch.sigmoid(fmap) * (1 - 2 * eps) + eps
    res = (res - 0.5) * 2
    return dict(rgb=res)

def postprocess_score(scores, eps=1e-6):
    out = torch.sigmoid(scores) * (1 - 2 * eps) + eps
    return out

def postprocess_pose(out, mode, inverse=False):
    """
    extract pose from prediction head output
    """
    mode, vmin, vmax = mode

    no_bounds = (vmin == -float("inf")) and (vmax == float("inf"))
    assert no_bounds
    trans = out[..., 0:3]
    quats = out[..., 3:7]

    if mode == "linear":
        if no_bounds:


def postprocess(out, depth_mode, conf_mode, pos_z=False):
    """
    extract 3D points/confidence from prediction head output
    """
    fmap = out.permute(0, 2, 3, 1)  # B,H,W,3
    res = dict(pts3d=reg_dense_depth(fmap[:, :, :, 0:3], mode=depth_mode, pos_z=pos_z))

    if conf_mode is not None:
        res["conf"] = reg_dense_conf(fmap[:, :, :, 3], mode=conf_mode)
    return res

def postprocess_rgb(out, eps=1e-6):
    fmap = out.permute(0, 2, 3, 1)  # B,H,W,3
    res = torch.sigmoid(fmap) * (1 - 2 * eps) + eps
    res = (res - 0.5) * 2
    return dict(rgb=res)

def postprocess_score(scores, eps=1e-6):
    out = torch.sigmoid(scores) * (1 - 2 * eps) + eps
    return out

def postprocess_pose(out, mode, inverse=False):
    """
    extract pose from prediction head output
    """
    mode, vmin, vmax = mode

    no_bounds = (vmin == -float("inf")) and (vmax == float("inf"))
    assert no_bounds
    trans = out[..., 0:3]
    quats = out[..., 3:7]

    if mode == "linear":
        if no_bounds:
            return trans  # [-inf, +inf]
        return trans.clip(min=vmin, max=vmax)

    d = trans.norm(dim=-1, keepdim=True)


    fmap = out.permute(0, 2, 3, 1)  # B,H,W,3
    res = dict(pts3d=reg_dense_depth(fmap[:, :, :, 0:3], mode=depth_mode, pos_z=pos_z))

    if conf_mode is not None:
        res["conf"] = reg_dense_conf(fmap[:, :, :, 3], mode=conf_mode)
    return res

def postprocess_rgb(out, eps=1e-6):
    fmap = out.permute(0, 2, 3, 1)  # B,H,W,3
    res = torch.sigmoid(fmap) * (1 - 2 * eps) + eps
    res = (res - 0.5) * 2
    return dict(rgb=res)

def postprocess_score(scores, eps=1e-6):
    out = torch.sigmoid(scores) * (1 - 2 * eps) + eps
    return out

def postprocess_pose(out, mode, inverse=False):
    """
    extract pose from prediction head output
    """
    mode, vmin, vmax = mode

    no_bounds = (vmin == -float("inf")) and (vmax == float("inf"))
    assert no_bounds
    trans = out[..., 0:3]
    quats = out[..., 3:7]

    if mode == "linear":
        if no_bounds:
            return trans  # [-inf, +inf]
        return trans.clip(min=vmin, max=vmax)

    d = trans.norm(dim=-1, keepdim=True)

    if mode == "square":
        if inverse:
            scale = d / d.square().clip(min=1e-8)
        else:
            scale = d.square() / d.clip(min=1e-8)

    if mode == "exp":
        if inverse:
            scale = d / torch.expm1(d).clip(min=1e-8)
        else:
            scale = torch.expm1(d) / d.clip(min=1e-8)

    trans = trans * scale
    quats = standardize_quaternion(quats)

    return torch.cat([trans, quats], dim=-1)

def postprocess_smpl(out, mode, pos_z=False, naive_mode=False):
    """
    extract SMPL parameters from prediction head output
    """
    pred_body_pose, pred_betas, pred_cam, pred_expression = out
    bs, max_humans = pred_body_pose.shape[:2]

    res = dict(smpl_shape=pred_betas)
    if naive_mode:
        res['smpl_transl'] = pred_cam # MHMR
    else:
        res['smpl_transl'] = reg_dense_depth(pred_cam, mode=mode, pos_z=pos_z)

    res['smpl_rotmat'] = rot6d_to_rotmat(
        pred_body_pose.view(-1, pred_body_pose.shape[-1]),
        naive_mode=naive_mode,
        ).view(bs, max_humans, -1, 3, 3)
    res['smpl_expression'] = pred_expression


    d = trans.norm(dim=-1, keepdim=True)

    if mode == "square":
        if inverse:
            scale = d / d.square().clip(min=1e-8)
        else:
            scale = d.square() / d.clip(min=1e-8)

    if mode == "exp":
        if inverse:
            scale = d / torch.expm1(d).clip(min=1e-8)
        else:
            scale = torch.expm1(d) / d.clip(min=1e-8)

    trans = trans * scale
    quats = standardize_quaternion(quats)

    return torch.cat([trans, quats], dim=-1)

def postprocess_smpl(out, mode, pos_z=False, naive_mode=False):
    """
    extract SMPL parameters from prediction head output
    """
    pred_body_pose, pred_betas, pred_cam, pred_expression = out
    bs, max_humans = pred_body_pose.shape[:2]

    res = dict(smpl_shape=pred_betas)
    if naive_mode:
        res['smpl_transl'] = pred_cam # MHMR
    else:
        res['smpl_transl'] = reg_dense_depth(pred_cam, mode=mode, pos_z=pos_z)

    res['smpl_rotmat'] = rot6d_to_rotmat(
        pred_body_pose.view(-1, pred_body_pose.shape[-1]),
        naive_mode=naive_mode,
        ).view(bs, max_humans, -1, 3, 3)
    res['smpl_expression'] = pred_expression

    return res

def postprocess_pose_conf(out):
    fmap = out.permute(0, 2, 3, 1)  # B,H,W,1
    return dict(pose_conf=torch.sigmoid(fmap))

def postprocess_desc(out, depth_mode, conf_mode, desc_dim, double_channel=False):
    """
    extract 3D points/confidence from prediction head output
    """
    fmap = out.permute(0, 2, 3, 1)  # B,H,W,3
    res = dict(pts3d=reg_dense_depth(fmap[:, :, :, 0:3], mode=depth_mode))

    if conf_mode is not None:
        res["conf"] = reg_dense_conf(fmap[:, :, :, 3], mode=conf_mode)

    if double_channel:
        res["pts3d_self"] = reg_dense_depth(
            fmap[


    extract SMPL parameters from prediction head output
    """
    pred_body_pose, pred_betas, pred_cam, pred_expression = out
    bs, max_humans = pred_body_pose.shape[:2]

    res = dict(smpl_shape=pred_betas)
    if naive_mode:
        res['smpl_transl'] = pred_cam # MHMR
    else:
        res['smpl_transl'] = reg_dense_depth(pred_cam, mode=mode, pos_z=pos_z)

    res['smpl_rotmat'] = rot6d_to_rotmat(
        pred_body_pose.view(-1, pred_body_pose.shape[-1]),
        naive_mode=naive_mode,
        ).view(bs, max_humans, -1, 3, 3)
    res['smpl_expression'] = pred_expression

    return res

def postprocess_pose_conf(out):
    fmap = out.permute(0, 2, 3, 1)  # B,H,W,1
    return dict(pose_conf=torch.sigmoid(fmap))

def postprocess_desc(out, depth_mode, conf_mode, desc_dim, double_channel=False):
    """
    extract 3D points/confidence from prediction head output
    """
    fmap = out.permute(0, 2, 3, 1)  # B,H,W,3
    res = dict(pts3d=reg_dense_depth(fmap[:, :, :, 0:3], mode=depth_mode))

    if conf_mode is not None:
        res["conf"] = reg_dense_conf(fmap[:, :, :, 3], mode=conf_mode)

    if double_channel:
        res["pts3d_self"] = reg_dense_depth(
            fmap[
                :, :, :, 3 + int(conf_mode is not None) : 6 + int(conf_mode is not None)
            ],
            mode=depth_mode,
        )
        if conf_mode is not None:


    res = dict(smpl_shape=pred_betas)
    if naive_mode:
        res['smpl_transl'] = pred_cam # MHMR
    else:
        res['smpl_transl'] = reg_dense_depth(pred_cam, mode=mode, pos_z=pos_z)

    res['smpl_rotmat'] = rot6d_to_rotmat(
        pred_body_pose.view(-1, pred_body_pose.shape[-1]),
        naive_mode=naive_mode,
        ).view(bs, max_humans, -1, 3, 3)
    res['smpl_expression'] = pred_expression

    return res

def postprocess_pose_conf(out):
    fmap = out.permute(0, 2, 3, 1)  # B,H,W,1
    return dict(pose_conf=torch.sigmoid(fmap))

def postprocess_desc(out, depth_mode, conf_mode, desc_dim, double_channel=False):
    """
    extract 3D points/confidence from prediction head output
    """
    fmap = out.permute(0, 2, 3, 1)  # B,H,W,3
    res = dict(pts3d=reg_dense_depth(fmap[:, :, :, 0:3], mode=depth_mode))

    if conf_mode is not None:
        res["conf"] = reg_dense_conf(fmap[:, :, :, 3], mode=conf_mode)

    if double_channel:
        res["pts3d_self"] = reg_dense_depth(
            fmap[
                :, :, :, 3 + int(conf_mode is not None) : 6 + int(conf_mode is not None)
            ],
            mode=depth_mode,
        )
        if conf_mode is not None:
            res["conf_self"] = reg_dense_conf(
                fmap[:, :, :, 6 + int(conf_mode is not None)], mode=conf_mode
            )

    start = (
        3
        + int(conf_mode is not None)
        + int(double_channel) * (3 + int(conf_mode is not None))
    )
    res["desc"] = reg_desc(fmap[:, :, :, start : start + desc_dim], mode="norm")
    res["desc_conf"] = reg_dense_conf(fmap[:, :, :, start + desc_dim], mode=conf_mode)
    assert start + desc_dim + 1 == fmap.shape[-1]

    return res

def reg_desc(desc, mode="norm"):
    if "norm" in mode:
        desc = desc / desc.norm(dim=-1, keepdim=True)
    else:
        raise ValueError(f"Unknown desc mode {mode}")
    return desc

def reg_dense_depth(xyz, mode, pos_z=False):
    """
    extract 3D points from prediction head output
    """
    mode, vmin, vmax = mode

    no_bounds = (vmin == -float("inf")) and (vmax == float("inf"))
    assert no_bounds

    if mode == "linear":


            ],
            mode=depth_mode,
        )
        if conf_mode is not None:
            res["conf_self"] = reg_dense_conf(
                fmap[:, :, :, 6 + int(conf_mode is not None)], mode=conf_mode
            )

    start = (
        3
        + int(conf_mode is not None)
        + int(double_channel) * (3 + int(conf_mode is not None))
    )
    res["desc"] = reg_desc(fmap[:, :, :, start : start + desc_dim], mode="norm")
    res["desc_conf"] = reg_dense_conf(fmap[:, :, :, start + desc_dim], mode=conf_mode)
    assert start + desc_dim + 1 == fmap.shape[-1]

    return res

def reg_desc(desc, mode="norm"):
    if "norm" in mode:
        desc = desc / desc.norm(dim=-1, keepdim=True)
    else:
        raise ValueError(f"Unknown desc mode {mode}")
    return desc

def reg_dense_depth(xyz, mode, pos_z=False):
    """
    extract 3D points from prediction head output
    """
    mode, vmin, vmax = mode

    no_bounds = (vmin == -float("inf")) and (vmax == float("inf"))
    assert no_bounds

    if mode == "linear":
        if no_bounds:
            return xyz  # [-inf, +inf]
        return xyz.clip(min=vmin, max=vmax)

    if pos_z:
        sign = torch.sign(xyz[..., -1:])
        xyz *= sign
    d = xyz.norm(dim=-1, keepdim=True)


    start = (
        3
        + int(conf_mode is not None)
        + int(double_channel) * (3 + int(conf_mode is not None))
    )
    res["desc"] = reg_desc(fmap[:, :, :, start : start + desc_dim], mode="norm")
    res["desc_conf"] = reg_dense_conf(fmap[:, :, :, start + desc_dim], mode=conf_mode)
    assert start + desc_dim + 1 == fmap.shape[-1]

    return res

def reg_desc(desc, mode="norm"):
    if "norm" in mode:
        desc = desc / desc.norm(dim=-1, keepdim=True)
    else:
        raise ValueError(f"Unknown desc mode {mode}")
    return desc

def reg_dense_depth(xyz, mode, pos_z=False):
    """
    extract 3D points from prediction head output
    """
    mode, vmin, vmax = mode

    no_bounds = (vmin == -float("inf")) and (vmax == float("inf"))
    assert no_bounds

    if mode == "linear":
        if no_bounds:
            return xyz  # [-inf, +inf]
        return xyz.clip(min=vmin, max=vmax)

    if pos_z:
        sign = torch.sign(xyz[..., -1:])
        xyz *= sign
    d = xyz.norm(dim=-1, keepdim=True)
    xyz = xyz / d.clip(min=1e-8)

    if mode == "square":
        return xyz * d.square()

    if mode == "exp":
        return xyz * torch.expm1(d)

    raise ValueError(f"bad {mode=}")

def reg_dense_conf(x, mode):
    """
    extract confidence from prediction head output
    """
    mode, vmin, vmax = mode
    if mode == "exp":
        return vmin + x.exp().clip(max=vmax - vmin)
    if mode == "sigmoid":
        return (vmax - vmin) * torch.sigmoid(x) + vmin
    raise ValueError(f"bad {mode=}")

def standardize_quaternion(quaternions: torch.Tensor) -> torch.Tensor:
    """
    Convert a unit quaternion to a standard form: one in which the real
    part is non negative.

    Args:


    if mode == "linear":
        if no_bounds:
            return xyz  # [-inf, +inf]
        return xyz.clip(min=vmin, max=vmax)

    if pos_z:
        sign = torch.sign(xyz[..., -1:])
        xyz *= sign
    d = xyz.norm(dim=-1, keepdim=True)
    xyz = xyz / d.clip(min=1e-8)

    if mode == "square":
        return xyz * d.square()

    if mode == "exp":
        return xyz * torch.expm1(d)

    raise ValueError(f"bad {mode=}")

def reg_dense_conf(x, mode):
    """
    extract confidence from prediction head output
    """
    mode, vmin, vmax = mode
    if mode == "exp":
        return vmin + x.exp().clip(max=vmax - vmin)
    if mode == "sigmoid":
        return (vmax - vmin) * torch.sigmoid(x) + vmin
    raise ValueError(f"bad {mode=}")

def standardize_quaternion(quaternions: torch.Tensor) -> torch.Tensor:
    """
    Convert a unit quaternion to a standard form: one in which the real
    part is non negative.

    Args:
        quaternions: Quaternions with real part first,
            as tensor of shape (..., 4).

    Returns:
        Standardized quaternions as tensor of shape (..., 4).
    """
    quaternions = F.normalize(quaternions, p=2, dim=-1)
    return torch.where(quaternions[..., 0:1] < 0, -quaternions, quaternions)

def rot6d_to_rotmat(x, naive_mode=False):
    """


        return xyz * d.square()

    if mode == "exp":
        return xyz * torch.expm1(d)

    raise ValueError(f"bad {mode=}")

def reg_dense_conf(x, mode):
    """
    extract confidence from prediction head output
    """
    mode, vmin, vmax = mode
    if mode == "exp":
        return vmin + x.exp().clip(max=vmax - vmin)
    if mode == "sigmoid":
        return (vmax - vmin) * torch.sigmoid(x) + vmin
    raise ValueError(f"bad {mode=}")

def standardize_quaternion(quaternions: torch.Tensor) -> torch.Tensor:
    """
    Convert a unit quaternion to a standard form: one in which the real
    part is non negative.

    Args:
        quaternions: Quaternions with real part first,
            as tensor of shape (..., 4).

    Returns:
        Standardized quaternions as tensor of shape (..., 4).
    """
    quaternions = F.normalize(quaternions, p=2, dim=-1)
    return torch.where(quaternions[..., 0:1] < 0, -quaternions, quaternions)

def rot6d_to_rotmat(x, naive_mode=False):
    """
    6D rotation representation to 3x3 rotation matrix.
    Args:
        x: (nvh,num_joints*6) Batch of 6-D rotation representations.
    Returns:
        torch.Tensor: Batch of corresponding rotation matrices with shape (nvh*num_joints,3,3).
    """
    if naive_mode:
        x = x.reshape(-1,2,3).permute(0, 2, 1).contiguous() # (nvh, num_joints*6) -> (nvh*num_joints, 3, 2) # inherited from MHMR
    else:
        x = x.reshape(-1,3,2).contiguous() # (nvh, num_joints*6) -> (nvh*num_joints, 3, 2)
    y = roma.special_gramschmidt(x, epsilon=1e-6) # (nvh*num_joints, 3, 3)
    return y


        return (vmax - vmin) * torch.sigmoid(x) + vmin
    raise ValueError(f"bad {mode=}")

def standardize_quaternion(quaternions: torch.Tensor) -> torch.Tensor:
    """
    Convert a unit quaternion to a standard form: one in which the real
    part is non negative.

    Args:
        quaternions: Quaternions with real part first,
            as tensor of shape (..., 4).

    Returns:
        Standardized quaternions as tensor of shape (..., 4).
    """
    quaternions = F.normalize(quaternions, p=2, dim=-1)
    return torch.where(quaternions[..., 0:1] < 0, -quaternions, quaternions)

def rot6d_to_rotmat(x, naive_mode=False):
    """
    6D rotation representation to 3x3 rotation matrix.
    Args:
        x: (nvh,num_joints*6) Batch of 6-D rotation representations.
    Returns:
        torch.Tensor: Batch of corresponding rotation matrices with shape (nvh*num_joints,3,3).
    """
    if naive_mode:
        x = x.reshape(-1,2,3).permute(0, 2, 1).contiguous() # (nvh, num_joints*6) -> (nvh*num_joints, 3, 2) # inherited from MHMR
    else:
        x = x.reshape(-1,3,2).contiguous() # (nvh, num_joints*6) -> (nvh*num_joints, 3, 2)
    y = roma.special_gramschmidt(x, epsilon=1e-6) # (nvh*num_joints, 3, 3)
    return y


# Copyright (C) 2024-present Naver Corporation. All rights reserved.
# Licensed under CC BY-NC-SA 4.0 (non-commercial use only).
#
# --------------------------------------------------------
# modified from DUSt3R

import torch
import torch.nn as nn
import torch.nn.functional as F
from dust3r.heads.postprocess import (
    postprocess,
    postprocess_desc,
    postprocess_rgb,
    postprocess_pose_conf,
    postprocess_pose,
    reg_dense_conf,
)
import dust3r.utils.path_to_croco  # noqa
from models.blocks import Mlp  # noqa
from dust3r.utils.geometry import geotrf
from dust3r.utils.camera import pose_encoding_to_camera, PoseDecoder
from dust3r.blocks import ConditionModulationBlock

class LinearPts3d(nn.Module):
    """
    Linear head for dust3r
    Each token outputs: - 16x16 3D points (+ confidence)
    """

    def __init__(
        self, net, has_conf=False, has_depth=False, has_rgb=False, has_pose_conf=False
    ):
        super().__init__()
        self.patch_size = net.patch_embed.patch_size[0]
        self.depth_mode = net.depth_mode
        self.conf_mode = net.conf_mode
        self.has_conf = has_conf
        self.has_rgb = has_rgb
        self.has_pose_conf = has_pose_conf
        self.has_depth = has_depth
        self.proj = Mlp(
            net.dec_embed_dim, out_features=(3 + has_conf) * self.patch_size**2
        )
        if has_depth:
            self.self_proj = Mlp(
                net.dec_embed_dim, out_features=(3 + has_conf) * self.patch_size**2
            )
        if has_rgb:
            self.rgb_proj = Mlp(net.dec_embed_dim, out_features=3 * self.patch_size**2)

    def setup(self, croconet):
        pass

    def forward(self, decout, img_shape):
        H, W = img_shape
        tokens = decout[-1]
        B, S, D = tokens.shape

        feat = self.proj(tokens)  # B,S,D
        feat = feat.transpose(-1, -2).view(
            B, -1, H // self.patch_size, W // self.patch_size
        )
        feat = F.pixel_shuffle(feat, self.patch_size)  # B,3,H,W

        final_output = postprocess(feat, self.depth_mode, self.conf_mode)
        final_output["pts3d_in_other_view"] = final_output.pop("pts3d")

        if self.has_depth:
            self_feat = self.self_proj(tokens)  # B,S,D
            self_feat = self_feat.transpose(-1, -2).view(
                B, -1, H // self.patch_size, W // self.patch_size
            )
            self_feat = F.pixel_shuffle(self_feat, self.patch_size)  # B,3,H,W
            self_3d_output = postprocess(self_feat, self.depth_mode, self.conf_mode)
            self_3d_output["pts3d_in_self_view"] = self_3d_output.pop("pts3d")
            self_3d_output["conf_self"] = self_3d_output.pop("conf")
            final_output.update(self_3d_output)

        if self.has_rgb:
            rgb_feat = self.rgb_proj(tokens)
            rgb_feat = rgb_feat.transpose(-1, -2).view(
                B, -1, H // self.patch_size, W // self.patch_size
            )
            rgb_feat = F.pixel_shuffle(rgb_feat, self.patch_size)  # B,3,H,W
            rgb_output = postprocess_rgb(rgb_feat)
            final_output.update(rgb_output)

        if self.has_pose_conf:
            pose_conf = self.pose_conf_proj(tokens)
            pose_conf = pose_conf.transpose(-1, -2).view(
                B, -1, H // self.patch_size, W // self.patch_size
            )
            pose_conf = F.pixel_shuffle(pose_conf, self.patch_size)
            pose_conf_output = postprocess_pose_conf(pose_conf)
            final_output.update(pose_conf_output)

        return final_output

class LinearPts3d_Desc(nn.Module):
    """
    Linear head for dust3r
    Each token outputs: - 16x16 3D points (+ confidence)
    """

    def __init__(
        self,
        net,
        has_conf=False,
        has_depth=False,
        local_feat_dim=24,
        hidden_dim_factor=4.0,
    ):
        super().__init__()
        self.patch_size = net.patch_embed.patch_size[0]
        self.depth_mode = net.depth_mode
        self.conf_mode = net.conf_mode
        self.has_conf = has_conf
        self.double_channel = has_depth
        self.local_feat_dim = local_feat_dim

        if not has_depth:
            self.proj = nn.Linear(
                net.dec_embed_dim, (3 + has_conf) * self.patch_size**2
            )
        else:
            self.proj = nn.Linear(
                net.dec_embed_dim, (3 + has_conf) * 2 * self.patch_size**2
            )
        idim = net.enc_embed_dim + net.dec_embed_dim
        self.head_local_features = Mlp(
            in_features=idim,
            hidden_features=int(hidden_dim_factor * idim),
            out_features=(self.local_feat_dim + 1) * self.patch_size**2,
        )

    def setup(self, croconet):
        pass

    def forward(self, decout, img_shape):
        H, W = img_shape
        tokens = decout[-1]
        B, S, D = tokens.shape

        feat = self.proj(tokens)  # B,S,D
        feat = feat.transpose(-1, -2).view(
            B, -1, H // self.patch_size, W // self.patch_size
        )
        feat = F.pixel_shuffle(feat, self.patch_size)  # B,3,H,W

        enc_output, dec_output = decout[0], decout[-1]
        cat_output = torch.cat([enc_output, dec_output], dim=-1)
        local_features = self.head_local_features(cat_output)  # B,S,D
        local_features = local_features.transpose(-1, -2).view(
            B, -1, H // self.patch_size, W // self.patch_size
        )
        local_features = F.pixel_shuffle(local_features, self.patch_size)  # B,d,H,W
        feat = torch.cat([feat, local_features], dim=1)

        return postprocess_desc(
            feat,
            self.depth_mode,
            self.conf_mode,
            self.local_feat_dim,
            self.double_channel,
        )

class LinearPts3dPoseDirect(nn.Module):
    """
    Linear head for dust3r
    Each token outputs: - 16x16 3D points (+ confidence)
    """

    def __init__(self, net, has_conf=False, has_rgb=False, has_pose=False):
        super().__init__()
        self.patch_size = net.patch_embed.patch_size[0]
        self.depth_mode = net.depth_mode
        self.conf_mode = net.conf_mode
        self.pose_mode = net.pose_mode
        self.has_conf = has_conf
        self.has_rgb = has_rgb
        self.has_pose = has_pose

        self.proj = Mlp(
            net.dec_embed_dim, out_features=(3 + has_conf) * self.patch_size**2
        )
        if has_rgb:
            self.rgb_proj = Mlp(net.dec_embed_dim, out_features=3 * self.patch_size**2)
        if has_pose:
            self.pose_head = PoseDecoder(hidden_size=net.dec_embed_dim)
        if has_conf:
            self.cross_conf_proj = Mlp(
                net.dec_embed_dim, out_features=self.patch_size**2
            )

    def setup(self, croconet):
        pass

    def forward(self, decout, img_shape):
        H, W = img_shape
        tokens = decout[-1]
        if self.has_pose:
            pose_token = tokens[:, 0]
            tokens = tokens[:, 1:]
        B, S, D = tokens.shape

        feat = self.proj(tokens)  # B,S,D
        feat = feat.transpose(-1, -2).view(
            B, -1, H // self.patch_size, W // self.patch_size
        )
        feat = F.pixel_shuffle(feat, self.patch_size)  # B,3,H,W
        final_output = postprocess(feat, self.depth_mode, self.conf_mode)
        final_output["pts3d_in_self_view"] = final_output.pop("pts3d")
        final_output["conf_self"] = final_output.pop("conf")

        if self.has_rgb:
            rgb_feat = self.rgb_proj(tokens)
            rgb_feat = rgb_feat.transpose(-1, -2).view(
                B, -1, H // self.patch_size, W // self.patch_size
            )
            rgb_feat = F.pixel_shuffle(rgb_feat, self.patch_size)  # B,3,H,W
            rgb_output = postprocess_rgb(rgb_feat)
            final_output.update(rgb_output)

        if self.has_pose:
            pose = self.pose_head(pose_token)
            pose = postprocess_pose(pose, self.pose_mode)
            final_output["camera_pose"] = pose  # B,7
            final_output["pts3d_in_other_view"] = geotrf(
                pose_encoding_to_camera(final_output["camera_pose"]),
                final_output["pts3d_in_self_view"],
            )

        if self.has_conf:
            cross_conf = self.cross_conf_proj(tokens)
            cross_conf = cross_conf.transpose(-1, -2).view(
                B, -1, H // self.patch_size, W // self.patch_size
            )
            cross_conf = F.pixel_shuffle(cross_conf, self.patch_size)[:, 0]
            final_output["conf"] = reg_dense_conf(cross_conf, mode=self.conf_mode)
        return final_output

class LinearPts3dPose(nn.Module):
    """
    Linear head for dust3r
    Each token outputs: - 16x16 3D points (+ confidence)
    """

    def __init__(
        self, net, has_conf=False, has_rgb=False, has_pose=False, mlp_ratio=4.0
    ):
        super().__init__()
        self.patch_size = net.patch_embed.patch_size[0]
        self.depth_mode = net.depth_mode
        self.conf_mode = net.conf_mode
        self.pose_mode = net.pose_mode
        self.has_conf = has_conf
        self.has_rgb = has_rgb
        self.has_pose = has_pose

        self.proj = Mlp(
            net.dec_embed_dim,
            hidden_features=int(mlp_ratio * net.dec_embed_dim),
            out_features=(3 + has_conf) * self.patch_size**2,
        )
        if has_rgb:
            self.rgb_proj = Mlp(
                net.dec_embed_dim,
                hidden_features=int(mlp_ratio * net.dec_embed_dim),
                out_features=3 * self.patch_size**2,
            )
        if has_pose:
            self.pose_head = PoseDecoder(hidden_size=net.dec_embed_dim)
            self.final_transform = nn.ModuleList(
                [
                    ConditionModulationBlock(
                        net.dec_embed_dim,
                        net.dec_num_heads,
                        mlp_ratio=4.0,
                        qkv_bias=True,
                        rope=net.rope,
                    )
                    for _ in range(2)
                ]
            )
            self.cross_proj = Mlp(
                net.dec_embed_dim,
                hidden_features=int(mlp_ratio * net.dec_embed_dim),
                out_features=(3 + has_conf) * self.patch_size**2,
            )

    def setup(self, croconet):
        pass

    def forward(self, decout, img_shape, **kwargs):
        H, W = img_shape
        tokens = decout[-1]
        if self.has_pose:
            pose_token = tokens[:, 0]
            tokens = tokens[:, 1:]
            with torch.cuda.amp.autocast(enabled=False):
                pose = self.pose_head(pose_token)
            cross_tokens = tokens
            for blk in self.final_transform:
                cross_tokens = blk(cross_tokens, pose_token, kwargs.get("pos"))

        with torch.cuda.amp.autocast(enabled=False):
            B, S, D = tokens.shape

            feat = self.proj(tokens)  # B,S,D
            feat = feat.transpose(-1, -2).view(
                B, -1, H // self.patch_size, W // self.patch_size
            )
            feat = F.pixel_shuffle(feat, self.patch_size)  # B,3,H,W
            final_output = postprocess(
                feat, self.depth_mode, self.conf_mode, pos_z=True
            )
            final_output["pts3d_in_self_view"] = final_output.pop("pts3d")
            final_output["conf_self"] = final_output.pop("conf")

            if self.has_rgb:
                rgb_feat = self.rgb_proj(tokens)
                rgb_feat = rgb_feat.transpose(-1, -2).view(
                    B, -1, H // self.patch_size, W // self.patch_size
                )
                rgb_feat = F.pixel_shuffle(rgb_feat, self.patch_size)  # B,3,H,W
                rgb_output = postprocess_rgb(rgb_feat)
                final_output.update(rgb_output)

            if self.has_pose:
                pose = postprocess_pose(pose, self.pose_mode)
                final_output["camera_pose"] = pose  # B,7

                cross_feat = self.cross_proj(cross_tokens)  # B,S,D
                cross_feat = cross_feat.transpose(-1, -2).view(
                    B, -1, H // self.patch_size, W // self.patch_size
                )
                cross_feat = F.pixel_shuffle(cross_feat, self.patch_size)  # B,3,H,W
                tmp = postprocess(cross_feat, self.depth_mode, self.conf_mode)
                final_output["pts3d_in_other_view"] = tmp.pop("pts3d")
                final_output["conf"] = tmp.pop("conf")

            return final_output


# modified from DUSt3R

import torch
import torch.nn as nn
import torch.nn.functional as F
from dust3r.heads.postprocess import (
    postprocess,
    postprocess_desc,
    postprocess_rgb,
    postprocess_pose_conf,
    postprocess_pose,
    reg_dense_conf,
)
import dust3r.utils.path_to_croco  # noqa
from models.blocks import Mlp  # noqa
from dust3r.utils.geometry import geotrf
from dust3r.utils.camera import pose_encoding_to_camera, PoseDecoder
from dust3r.blocks import ConditionModulationBlock

class LinearPts3d(nn.Module):
    """
    Linear head for dust3r
    Each token outputs: - 16x16 3D points (+ confidence)
    """

    def __init__(
        self, net, has_conf=False, has_depth=False, has_rgb=False, has_pose_conf=False
    ):
        super().__init__()
        self.patch_size = net.patch_embed.patch_size[0]
        self.depth_mode = net.depth_mode
        self.conf_mode = net.conf_mode
        self.has_conf = has_conf
        self.has_rgb = has_rgb
        self.has_pose_conf = has_pose_conf
        self.has_depth = has_depth
        self.proj = Mlp(
            net.dec_embed_dim, out_features=(3 + has_conf) * self.patch_size**2
        )
        if has_depth:
            self.self_proj = Mlp(
                net.dec_embed_dim, out_features=(3 + has_conf) * self.patch_size**2
            )
        if has_rgb:
            self.rgb_proj = Mlp(net.dec_embed_dim, out_features=3 * self.patch_size**2)

    def setup(self, croconet):
        pass

    def forward(self, decout, img_shape):
        H, W = img_shape
        tokens = decout[-1]
        B, S, D = tokens.shape

        feat = self.proj(tokens)  # B,S,D
        feat = feat.transpose(-1, -2).view(
            B, -1, H // self.patch_size, W // self.patch_size
        )
        feat = F.pixel_shuffle(feat, self.patch_size)  # B,3,H,W

        final_output = postprocess(feat, self.depth_mode, self.conf_mode)
        final_output["pts3d_in_other_view"] = final_output.pop("pts3d")

        if self.has_depth:
            self_feat = self.self_proj(tokens)  # B,S,D
            self_feat = self_feat.transpose(-1, -2).view(
                B, -1, H // self.patch_size, W // self.patch_size
            )
            self_feat = F.pixel_shuffle(self_feat, self.patch_size)  # B,3,H,W
            self_3d_output = postprocess(self_feat, self.depth_mode, self.conf_mode)
            self_3d_output["pts3d_in_self_view"] = self_3d_output.pop("pts3d")
            self_3d_output["conf_self"] = self_3d_output.pop("conf")
            final_output.update(self_3d_output)

        if self.has_rgb:
            rgb_feat = self.rgb_proj(tokens)
            rgb_feat = rgb_feat.transpose(-1, -2).view(
                B, -1, H // self.patch_size, W // self.patch_size
            )
            rgb_feat = F.pixel_shuffle(rgb_feat, self.patch_size)  # B,3,H,W
            rgb_output = postprocess_rgb(rgb_feat)
            final_output.update(rgb_output)

        if self.has_pose_conf:
            pose_conf = self.pose_conf_proj(tokens)
            pose_conf = pose_conf.transpose(-1, -2).view(
                B, -1, H // self.patch_size, W // self.patch_size
            )
            pose_conf = F.pixel_shuffle(pose_conf, self.patch_size)
            pose_conf_output = postprocess_pose_conf(pose_conf)
            final_output.update(pose_conf_output)

        return final_output

class LinearPts3d_Desc(nn.Module):
    """
    Linear head for dust3r
    Each token outputs: - 16x16 3D points (+ confidence)
    """

    def __init__(
        self,
        net,
        has_conf=False,
        has_depth=False,
        local_feat_dim=24,
        hidden_dim_factor=4.0,
    ):
        super().__init__()
        self.patch_size = net.patch_embed.patch_size[0]
        self.depth_mode = net.depth_mode
        self.conf_mode = net.conf_mode


            rgb_feat = self.rgb_proj(tokens)
            rgb_feat = rgb_feat.transpose(-1, -2).view(
                B, -1, H // self.patch_size, W // self.patch_size
            )
            rgb_feat = F.pixel_shuffle(rgb_feat, self.patch_size)  # B,3,H,W
            rgb_output = postprocess_rgb(rgb_feat)
            final_output.update(rgb_output)

        if self.has_pose_conf:
            pose_conf = self.pose_conf_proj(tokens)
            pose_conf = pose_conf.transpose(-1, -2).view(
                B, -1, H // self.patch_size, W // self.patch_size
            )
            pose_conf = F.pixel_shuffle(pose_conf, self.patch_size)
            pose_conf_output = postprocess_pose_conf(pose_conf)
            final_output.update(pose_conf_output)

        return final_output

class LinearPts3d_Desc(nn.Module):
    """
    Linear head for dust3r
    Each token outputs: - 16x16 3D points (+ confidence)
    """

    def __init__(
        self,
        net,
        has_conf=False,
        has_depth=False,
        local_feat_dim=24,
        hidden_dim_factor=4.0,
    ):
        super().__init__()
        self.patch_size = net.patch_embed.patch_size[0]
        self.depth_mode = net.depth_mode
        self.conf_mode = net.conf_mode
        self.has_conf = has_conf
        self.double_channel = has_depth
        self.local_feat_dim = local_feat_dim

        if not has_depth:
            self.proj = nn.Linear(
                net.dec_embed_dim, (3 + has_conf) * self.patch_size**2
            )
        else:
            self.proj = nn.Linear(
                net.dec_embed_dim, (3 + has_conf) * 2 * self.patch_size**2
            )
        idim = net.enc_embed_dim + net.dec_embed_dim
        self.head_local_features = Mlp(
            in_features=idim,
            hidden_features=int(hidden_dim_factor * idim),
            out_features=(self.local_feat_dim + 1) * self.patch_size**2,
        )

    def setup(self, croconet):
        pass

    def forward(self, decout, img_shape):
        H, W = img_shape
        tokens = decout[-1]
        B, S, D = tokens.shape

        feat = self.proj(tokens)  # B,S,D
        feat = feat.transpose(-1, -2).view(
            B, -1, H // self.patch_size, W // self.patch_size
        )
        feat = F.pixel_shuffle(feat, self.patch_size)  # B,3,H,W

        enc_output, dec_output = decout[0], decout[-1]
        cat_output = torch.cat([enc_output, dec_output], dim=-1)
        local_features = self.head_local_features(cat_output)  # B,S,D
        local_features = local_features.transpose(-1, -2).view(
            B, -1, H // self.patch_size, W // self.patch_size
        )
        local_features = F.pixel_shuffle(local_features, self.patch_size)  # B,d,H,W
        feat = torch.cat([feat, local_features], dim=1)

        return postprocess_desc(
            feat,
            self.depth_mode,
            self.conf_mode,
            self.local_feat_dim,
            self.double_channel,
        )

class LinearPts3dPoseDirect(nn.Module):
    """
    Linear head for dust3r
    Each token outputs: - 16x16 3D points (+ confidence)
    """

    def __init__(self, net, has_conf=False, has_rgb=False, has_pose=False):
        super().__init__()
        self.patch_size = net.patch_embed.patch_size[0]
        self.depth_mode = net.depth_mode
        self.conf_mode = net.conf_mode
        self.pose_mode = net.pose_mode
        self.has_conf = has_conf
        self.has_rgb = has_rgb
        self.has_pose = has_pose

        self.proj = Mlp(
            net.dec_embed_dim, out_features=(3 + has_conf) * self.patch_size**2


        feat = F.pixel_shuffle(feat, self.patch_size)  # B,3,H,W

        enc_output, dec_output = decout[0], decout[-1]
        cat_output = torch.cat([enc_output, dec_output], dim=-1)
        local_features = self.head_local_features(cat_output)  # B,S,D
        local_features = local_features.transpose(-1, -2).view(
            B, -1, H // self.patch_size, W // self.patch_size
        )
        local_features = F.pixel_shuffle(local_features, self.patch_size)  # B,d,H,W
        feat = torch.cat([feat, local_features], dim=1)

        return postprocess_desc(
            feat,
            self.depth_mode,
            self.conf_mode,
            self.local_feat_dim,
            self.double_channel,
        )

class LinearPts3dPoseDirect(nn.Module):
    """
    Linear head for dust3r
    Each token outputs: - 16x16 3D points (+ confidence)
    """

    def __init__(self, net, has_conf=False, has_rgb=False, has_pose=False):
        super().__init__()
        self.patch_size = net.patch_embed.patch_size[0]
        self.depth_mode = net.depth_mode
        self.conf_mode = net.conf_mode
        self.pose_mode = net.pose_mode
        self.has_conf = has_conf
        self.has_rgb = has_rgb
        self.has_pose = has_pose

        self.proj = Mlp(
            net.dec_embed_dim, out_features=(3 + has_conf) * self.patch_size**2
        )
        if has_rgb:
            self.rgb_proj = Mlp(net.dec_embed_dim, out_features=3 * self.patch_size**2)
        if has_pose:
            self.pose_head = PoseDecoder(hidden_size=net.dec_embed_dim)
        if has_conf:
            self.cross_conf_proj = Mlp(
                net.dec_embed_dim, out_features=self.patch_size**2
            )

    def setup(self, croconet):
        pass

    def forward(self, decout, img_shape):
        H, W = img_shape
        tokens = decout[-1]
        if self.has_pose:
            pose_token = tokens[:, 0]
            tokens = tokens[:, 1:]
        B, S, D = tokens.shape

        feat = self.proj(tokens)  # B,S,D
        feat = feat.transpose(-1, -2).view(
            B, -1, H // self.patch_size, W // self.patch_size
        )
        feat = F.pixel_shuffle(feat, self.patch_size)  # B,3,H,W
        final_output = postprocess(feat, self.depth_mode, self.conf_mode)
        final_output["pts3d_in_self_view"] = final_output.pop("pts3d")
        final_output["conf_self"] = final_output.pop("conf")

        if self.has_rgb:
            rgb_feat = self.rgb_proj(tokens)
            rgb_feat = rgb_feat.transpose(-1, -2).view(
                B, -1, H // self.patch_size, W // self.patch_size
            )
            rgb_feat = F.pixel_shuffle(rgb_feat, self.patch_size)  # B,3,H,W
            rgb_output = postprocess_rgb(rgb_feat)
            final_output.update(rgb_output)

        if self.has_pose:
            pose = self.pose_head(pose_token)
            pose = postprocess_pose(pose, self.pose_mode)
            final_output["camera_pose"] = pose  # B,7
            final_output["pts3d_in_other_view"] = geotrf(
                pose_encoding_to_camera(final_output["camera_pose"]),
                final_output["pts3d_in_self_view"],
            )

        if self.has_conf:
            cross_conf = self.cross_conf_proj(tokens)
            cross_conf = cross_conf.transpose(-1, -2).view(
                B, -1, H // self.patch_size, W // self.patch_size
            )
            cross_conf = F.pixel_shuffle(cross_conf, self.patch_size)[:, 0]
            final_output["conf"] = reg_dense_conf(cross_conf, mode=self.conf_mode)
        return final_output

class LinearPts3dPose(nn.Module):
    """
    Linear head for dust3r
    Each token outputs: - 16x16 3D points (+ confidence)
    """

    def __init__(
        self, net, has_conf=False, has_rgb=False, has_pose=False, mlp_ratio=4.0
    ):
        super().__init__()
        self.patch_size = net.patch_embed.patch_size[0]
        self.depth_mode = net.depth_mode
        self.conf_mode = net.conf_mode
        self.pose_mode = net.pose_mode
        self.has_conf = has_conf
        self.has_rgb = has_rgb
        self.has_pose = has_pose


        if self.has_pose:
            pose = self.pose_head(pose_token)
            pose = postprocess_pose(pose, self.pose_mode)
            final_output["camera_pose"] = pose  # B,7
            final_output["pts3d_in_other_view"] = geotrf(
                pose_encoding_to_camera(final_output["camera_pose"]),
                final_output["pts3d_in_self_view"],
            )

        if self.has_conf:
            cross_conf = self.cross_conf_proj(tokens)
            cross_conf = cross_conf.transpose(-1, -2).view(
                B, -1, H // self.patch_size, W // self.patch_size
            )
            cross_conf = F.pixel_shuffle(cross_conf, self.patch_size)[:, 0]
            final_output["conf"] = reg_dense_conf(cross_conf, mode=self.conf_mode)
        return final_output

class LinearPts3dPose(nn.Module):
    """
    Linear head for dust3r
    Each token outputs: - 16x16 3D points (+ confidence)
    """

    def __init__(
        self, net, has_conf=False, has_rgb=False, has_pose=False, mlp_ratio=4.0
    ):
        super().__init__()
        self.patch_size = net.patch_embed.patch_size[0]
        self.depth_mode = net.depth_mode
        self.conf_mode = net.conf_mode
        self.pose_mode = net.pose_mode
        self.has_conf = has_conf
        self.has_rgb = has_rgb
        self.has_pose = has_pose

        self.proj = Mlp(
            net.dec_embed_dim,
            hidden_features=int(mlp_ratio * net.dec_embed_dim),
            out_features=(3 + has_conf) * self.patch_size**2,
        )
        if has_rgb:
            self.rgb_proj = Mlp(
                net.dec_embed_dim,
                hidden_features=int(mlp_ratio * net.dec_embed_dim),
                out_features=3 * self.patch_size**2,
            )
        if has_pose:
            self.pose_head = PoseDecoder(hidden_size=net.dec_embed_dim)
            self.final_transform = nn.ModuleList(
                [
                    ConditionModulationBlock(
                        net.dec_embed_dim,
                        net.dec_num_heads,
                        mlp_ratio=4.0,
                        qkv_bias=True,
                        rope=net.rope,
                    )
                    for _ in range(2)
                ]
            )
            self.cross_proj = Mlp(
                net.dec_embed_dim,
                hidden_features=int(mlp_ratio * net.dec_embed_dim),
                out_features=(3 + has_conf) * self.patch_size**2,
            )

    def setup(self, croconet):
        pass

    def forward(self, decout, img_shape, **kwargs):
        H, W = img_shape
        tokens = decout[-1]
        if self.has_pose:
            pose_token = tokens[:, 0]
            tokens = tokens[:, 1:]
            with torch.cuda.amp.autocast(enabled=False):
                pose = self.pose_head(pose_token)
            cross_tokens = tokens
            for blk in self.final_transform:
                cross_tokens = blk(cross_tokens, pose_token, kwargs.get("pos"))

        with torch.cuda.amp.autocast(enabled=False):
            B, S, D = tokens.shape

            feat = self.proj(tokens)  # B,S,D
            feat = feat.transpose(-1, -2).view(
                B, -1, H // self.patch_size, W // self.patch_size
            )
            feat = F.pixel_shuffle(feat, self.patch_size)  # B,3,H,W
            final_output = postprocess(
                feat, self.depth_mode, self.conf_mode, pos_z=True
            )
            final_output["pts3d_in_self_view"] = final_output.pop("pts3d")
            final_output["conf_self"] = final_output.pop("conf")

            if self.has_rgb:
                rgb_feat = self.rgb_proj(tokens)
                rgb_feat = rgb_feat.transpose(-1, -2).view(
                    B, -1, H // self.patch_size, W // self.patch_size
                )
                rgb_feat = F.pixel_shuffle(rgb_feat, self.patch_size)  # B,3,H,W
                rgb_output = postprocess_rgb(rgb_feat)
                final_output.update(rgb_output)

            if self.has_pose:
                pose = postprocess_pose(pose, self.pose_mode)
                final_output["camera_pose"] = pose  # B,7

                cross_feat = self.cross_proj(cross_tokens)  # B,S,D
                cross_feat = cross_feat.transpose(-1, -2).view(
                    B, -1, H // self.patch_size, W // self.patch_size
                )
                cross_feat = F.pixel_shuffle(cross_feat, self.patch_size)  # B,3,H,W
                tmp = postprocess(cross_feat, self.depth_mode, self.conf_mode)
                final_output["pts3d_in_other_view"] = tmp.pop("pts3d")
                final_output["conf"] = tmp.pop("conf")

            return final_output


                pose_encoding_to_camera(final_output["camera_pose"]),
                final_output["pts3d_in_self_view"],
            )

        if self.has_conf:
            cross_conf = self.cross_conf_proj(tokens)
            cross_conf = cross_conf.transpose(-1, -2).view(
                B, -1, H // self.patch_size, W // self.patch_size
            )
            cross_conf = F.pixel_shuffle(cross_conf, self.patch_size)[:, 0]
            final_output["conf"] = reg_dense_conf(cross_conf, mode=self.conf_mode)
        return final_output

class LinearPts3dPose(nn.Module):
    """
    Linear head for dust3r
    Each token outputs: - 16x16 3D points (+ confidence)
    """

    def __init__(
        self, net, has_conf=False, has_rgb=False, has_pose=False, mlp_ratio=4.0
    ):
        super().__init__()
        self.patch_size = net.patch_embed.patch_size[0]
        self.depth_mode = net.depth_mode
        self.conf_mode = net.conf_mode
        self.pose_mode = net.pose_mode
        self.has_conf = has_conf
        self.has_rgb = has_rgb
        self.has_pose = has_pose

        self.proj = Mlp(
            net.dec_embed_dim,
            hidden_features=int(mlp_ratio * net.dec_embed_dim),
            out_features=(3 + has_conf) * self.patch_size**2,
        )
        if has_rgb:
            self.rgb_proj = Mlp(
                net.dec_embed_dim,
                hidden_features=int(mlp_ratio * net.dec_embed_dim),
                out_features=3 * self.patch_size**2,
            )
        if has_pose:
            self.pose_head = PoseDecoder(hidden_size=net.dec_embed_dim)
            self.final_transform = nn.ModuleList(
                [
                    ConditionModulationBlock(
                        net.dec_embed_dim,
                        net.dec_num_heads,
                        mlp_ratio=4.0,
                        qkv_bias=True,
                        rope=net.rope,
                    )
                    for _ in range(2)
                ]
            )
            self.cross_proj = Mlp(
                net.dec_embed_dim,
                hidden_features=int(mlp_ratio * net.dec_embed_dim),
                out_features=(3 + has_conf) * self.patch_size**2,
            )

    def setup(self, croconet):
        pass

    def forward(self, decout, img_shape, **kwargs):
        H, W = img_shape
        tokens = decout[-1]
        if self.has_pose:
            pose_token = tokens[:, 0]
            tokens = tokens[:, 1:]
            with torch.cuda.amp.autocast(enabled=False):
                pose = self.pose_head(pose_token)
            cross_tokens = tokens
            for blk in self.final_transform:
                cross_tokens = blk(cross_tokens, pose_token, kwargs.get("pos"))

        with torch.cuda.amp.autocast(enabled=False):
            B, S, D = tokens.shape

            feat = self.proj(tokens)  # B,S,D


setup

        if has_pose:
            self.pose_head = PoseDecoder(hidden_size=net.dec_embed_dim)
            self.final_transform = nn.ModuleList(
                [
                    ConditionModulationBlock(
                        net.dec_embed_dim,
                        net.dec_num_heads,
                        mlp_ratio=4.0,
                        qkv_bias=True,
                        rope=net.rope,
                    )
                    for _ in range(2)
                ]
            )
            self.cross_proj = Mlp(
                net.dec_embed_dim,
                hidden_features=int(mlp_ratio * net.dec_embed_dim),
                out_features=(3 + has_conf) * self.patch_size**2,
            )

    def setup(self, croconet):
        pass

    def forward(self, decout, img_shape, **kwargs):
        H, W = img_shape
        tokens = decout[-1]
        if self.has_pose:
            pose_token = tokens[:, 0]
            tokens = tokens[:, 1:]
            with torch.cuda.amp.autocast(enabled=False):
                pose = self.pose_head(pose_token)
            cross_tokens = tokens
            for blk in self.final_transform:
                cross_tokens = blk(cross_tokens, pose_token, kwargs.get("pos"))

        with torch.cuda.amp.autocast(enabled=False):
            B, S, D = tokens.shape

            feat = self.proj(tokens)  # B,S,D
            feat = feat.transpose(-1, -2).view(
                B, -1, H // self.patch_size, W // self.patch_size
            )


forward

                [
                    ConditionModulationBlock(
                        net.dec_embed_dim,
                        net.dec_num_heads,
                        mlp_ratio=4.0,
                        qkv_bias=True,
                        rope=net.rope,
                    )
                    for _ in range(2)
                ]
            )
            self.cross_proj = Mlp(
                net.dec_embed_dim,
                hidden_features=int(mlp_ratio * net.dec_embed_dim),
                out_features=(3 + has_conf) * self.patch_size**2,
            )

    def setup(self, croconet):
        pass

    def forward(self, decout, img_shape, **kwargs):
        H, W = img_shape
        tokens = decout[-1]
        if self.has_pose:
            pose_token = tokens[:, 0]
            tokens = tokens[:, 1:]
            with torch.cuda.amp.autocast(enabled=False):
                pose = self.pose_head(pose_token)
            cross_tokens = tokens
            for blk in self.final_transform:
                cross_tokens = blk(cross_tokens, pose_token, kwargs.get("pos"))

        with torch.cuda.amp.autocast(enabled=False):
            B, S, D = tokens.shape

            feat = self.proj(tokens)  # B,S,D
            feat = feat.transpose(-1, -2).view(
                B, -1, H // self.patch_size, W // self.patch_size
            )
            feat = F.pixel_shuffle(feat, self.patch_size)  # B,3,H,W
            final_output = postprocess(
                feat, self.depth_mode, self.conf_mode, pos_z=True
            )
            final_output["pts3d_in_self_view"] = final_output.pop("pts3d")
            final_output["conf_self"] = final_output.pop("conf")

            if self.has_rgb:
                rgb_feat = self.rgb_proj(tokens)
                rgb_feat = rgb_feat.transpose(-1, -2).view(
                    B, -1, H // self.patch_size, W // self.patch_size
                )
                rgb_feat = F.pixel_shuffle(rgb_feat, self.patch_size)  # B,3,H,W
                rgb_output = postprocess_rgb(rgb_feat)
                final_output.update(rgb_output)

            if self.has_pose:
                pose = postprocess_pose(pose, self.pose_mode)
                final_output["camera_pose"] = pose  # B,7

                cross_feat = self.cross_proj(cross_tokens)  # B,S,D
                cross_feat = cross_feat.transpose(-1, -2).view(
                    B, -1, H // self.patch_size, W // self.patch_size
                )
                cross_feat = F.pixel_shuffle(cross_feat, self.patch_size)  # B,3,H,W
                tmp = postprocess(cross_feat, self.depth_mode, self.conf_mode)
                final_output["pts3d_in_other_view"] = tmp.pop("pts3d")
                final_output["conf"] = tmp.pop("conf")

            return final_output


demo.py

#!/usr/bin/env python3
"""
Modified from CUT3R: https://github.com/CUT3R/CUT3R

Online Human-Scene Reconstruction Inference and Visualization Script

This script performs inference using the ARCroco3DStereo model and visualizes the
resulting 3D scene point clouds and SMPLX sequences with the SceneHumanViewer.
Use the command-line arguments to adjust parameters
such as the model checkpoint path, image sequence directory, image size, device, etc.

Example:
    python demo.py --model_path src/human3r.pth --size 512 \
        --seq_path examples/GoodMornin1.mp4 --subsample 1 --vis_threshold 2 \
        --downsample_factor 1 --use_ttt3r --reset_interval 100
"""

import os
import numpy as np
import torch
import time


import sys
import os
import os.path as path

def add_path_to_dust3r(ckpt):
    HERE_PATH = os.path.dirname(os.path.abspath(ckpt))
    # workaround for sibling import
    sys.path.insert(0, HERE_PATH)


import torch
import os
from matplotlib.figure import Figure
import matplotlib.pyplot as plt
from matplotlib.backends.backend_agg import FigureCanvasAgg
import matplotlib as mpl
import cv2
import numpy as np
import matplotlib.cm as cm
import viser
import viser.transforms as tf
import time
import trimesh
import dataclasses
from scipy.spatial.transform import Rotation
from skimage.morphology import binary_dilation, binary_erosion, disk
from src.dust3r.viz import (
    add_scene_cam,
    CAM_COLORS,
    OPENGL,
    pts3d_to_trimesh,


# Modified from Multi-HMR
# Preprocess 3DPW dataset for human evaluation, and saves the annotations
# files (i.e., 3dpw_test.pkl) in [ANNOT_DIR].
# Usage: python -m eval.dataset.prepare_3dpw "create_annots()"

import os
os.environ["PYOPENGL_PLATFORM"] = "egl"
os.environ['EGL_DEVICE_ID'] = '0'

import pickle
import torch
import smplx
from tqdm import tqdm
import sys
import numpy as np
from PIL import Image, ImageFile

sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))
sys.path.append(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), 'src'))
from dust3r.smpl_model import SMPLX_DIR


import os
import glob
from tqdm import tqdm

# Define the merged dataset metadata dictionary
dataset_metadata = {
    "davis": {
        "img_path": "data/davis/DAVIS/JPEGImages/480p",
        "mask_path": "data/davis/DAVIS/masked_images/480p",
        "dir_path_func": lambda img_path, seq: os.path.join(img_path, seq),
        "gt_traj_func": lambda img_path, anno_path, seq: None,
        "traj_format": None,
        "seq_list": None,
        "full_seq": True,
        "mask_path_seq_func": lambda mask_path, seq: os.path.join(mask_path, seq),
        "skip_condition": None,
        "process_func": None,  # Not used in mono depth estimation
    },
    "kitti": {
        "img_path": "data/kitti/depth_selection/val_selection_cropped/image_gathered",  # Default path
        "mask_path": None,


from copy import deepcopy
import cv2

import numpy as np
import torch
import torch.nn as nn
import roma
from copy import deepcopy
import tqdm
import matplotlib as mpl
import matplotlib.cm as cm
import matplotlib.pyplot as plt
from matplotlib.backends.backend_agg import FigureCanvasAgg
from scipy.spatial.transform import Rotation
from eval.relpose.evo_utils import *
from PIL import Image
import imageio.v2 as iio
from matplotlib.figure import Figure

# from checkpoints.dust3r.viz import colorize_np, colorize


import os
import sys

sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))
import math
import cv2
import numpy as np
import torch
import argparse

from copy import deepcopy
from eval.relpose.metadata import dataset_metadata
from eval.relpose.utils import *

from accelerate import PartialState
from add_ckpt_path import add_path_to_dust3r

from tqdm import tqdm

def get_args_parser():