model/base/correlation.py

r""" Provides functions that creates/manipulates correlation matrices """

import math

from torch.nn.functional import interpolate as resize
import torch

from .geometry import Geometry


class Correlation:

    @classmethod
    def mutual_nn_filter(cls, correlation_matrix, eps=1e-30):
        r""" Mutual nearest neighbor filtering (Rocco et al. NeurIPS'18 )"""
        corr_src_max = torch.max(correlation_matrix, dim=2, keepdim=True)[0]
        corr_trg_max = torch.max(correlation_matrix, dim=1, keepdim=True)[0]
        corr_src_max[corr_src_max == 0] += eps
        corr_trg_max[corr_trg_max == 0] += eps

        corr_src = correlation_matrix / corr_src_max
        corr_trg = correlation_matrix / corr_trg_max

        return correlation_matrix * (corr_src * corr_trg)

    @classmethod
    def build_correlation6d(self, src_feat, trg_feat, scales, conv2ds):
        r""" Build 6-dimensional correlation tensor """

        bsz, _, side, side = src_feat.size()

        # Construct feature pairs with multiple scales
        _src_feats = []
        _trg_feats = []
        for scale, conv in zip(scales, conv2ds):
            s = (round(side * math.sqrt(scale)),) * 2
            _src_feat = conv(resize(src_feat, s, mode='bilinear', align_corners=True))
            _trg_feat = conv(resize(trg_feat, s, mode='bilinear', align_corners=True))
            _src_feats.append(_src_feat)
            _trg_feats.append(_trg_feat)

        # Build multiple 4-dimensional correlation tensor
        corr6d = []
        for src_feat in _src_feats:
            ch = src_feat.size(1)

            src_side = src_feat.size(-1)
            src_feat = src_feat.view(bsz, ch, -1).transpose(1, 2)
            src_norm = src_feat.norm(p=2, dim=2, keepdim=True)

            for trg_feat in _trg_feats:
                trg_side = trg_feat.size(-1)
                trg_feat = trg_feat.view(bsz, ch, -1)
                trg_norm = trg_feat.norm(p=2, dim=1, keepdim=True)

                correlation = torch.bmm(src_feat, trg_feat) / torch.bmm(src_norm, trg_norm)
                correlation = correlation.view(bsz, src_side, src_side, trg_side, trg_side).contiguous()
                corr6d.append(correlation)

        # Resize the spatial sizes of the 4D tensors to the same size
        for idx, correlation in enumerate(corr6d):
            corr6d[idx] = Geometry.interpolate4d(correlation, [side, side])

        # Build 6-dimensional correlation tensor
        corr6d = torch.stack(corr6d).view(len(scales), len(scales),
                             bsz, side, side, side, side).permute(2, 0, 1, 3, 4, 5, 6)
        return corr6d.clamp(min=0)