Network/model/loss.py

import torch
from torch import nn
import math

import model.box_utils as box_utils
# fragment
# boundary

def cropped_box_iou(bboxes_pred,bboxes_gt,mask):
    H,W = mask.shape
    
    bboxes_pred = box_utils.norms_to_indices(bboxes_pred,W,H)
    bboxes_gt = box_utils.norms_to_indices(bboxes_gt,W,H)
    
    ious = torch.zeros(len(bboxes_pred)).to(bboxes_pred).float()
    for i in range(len(bboxes_pred)):
        box_pred = bboxes_pred[i]
        box_gt = bboxes_gt[i]

        mask_pred = mask.new_zeros(H,W)
        mask_gt = mask.new_zeros(H,W)

        mask_pred[box_pred[1]:box_pred[3],box_pred[0]:box_pred[2]]=1
        mask_gt[box_gt[1]:box_gt[3],box_gt[0]:box_gt[2]]=1

        mask_pred = mask_pred*mask
        mask_gt = mask_gt*mask

        union = (mask_pred+mask_gt)
        intersection = (mask_pred*mask_gt)
        
        ious[i] = len(intersection.nonzero())/len(union.nonzero())
    return ious

def compose_by_cateogry(boxes,objs,category):
    H,W = category.shape
    image = torch.zeros_like(category).to(category)
    boxes = box_utils.norms_to_indices(boxes,H,W)
    
    overlaps = category.new_zeros(H,W,len(boxes))
    for i,box in enumerate(boxes):
        #overlap = image[box[1]:box[3],box[0]:box[2]].nonzero()+box[[1,0]]
        overlaps[box[1]:box[3],box[0]:box[2],i] = 1
        image[box[1]:box[3],box[0]:box[2]] = objs[i]
            
    overlap_vectors = overlaps.view(-1,len(boxes)).unique(dim=0)
    for vector in overlap_vectors:
        if vector.sum()<2: continue
        overlap_region = (overlaps==vector).prod(-1).nonzero()
        unique,count = category[overlap_region[:,0],overlap_region[:,1]].unique(return_counts=True)
        overlap_objs = objs[vector.bool()]
        valid_u = [True if u in overlap_objs else False for u in unique]
        count = count[valid_u]
        unique = unique[valid_u]
        if len(unique)>0:
            winner = unique[count.argmax()]
            image[overlap_region[:,0],overlap_region[:,1]] = winner
    return image

def sample_fragment(step=2):
    """
    Parameters:
    ----------
    step: int, sample step in linspace [0,1]

    Returns:
    ----------
    ret: [step^step,2]
    """
    return torch.stack(
        torch.meshgrid(
            torch.linspace(0,1,step),
            torch.linspace(0,1,step)
        )
    ,dim=-1).reshape(-1,2)

def sample_boundary(step=2):
    """
    Parameters:
    ----------
    step: int, sample step in linspace [0,1]

    Returns:
    ----------
    ret: [step*4,2]
    """
    return torch.cat([
        torch.stack(torch.meshgrid(
            torch.linspace(0,1,2),
            torch.linspace(0,1,step)
        ),dim=-1).reshape(-1,2),
        torch.stack(torch.meshgrid(
            torch.linspace(0,1,step),
            torch.linspace(0,1,2)
        ),dim=-1).reshape(-1,2)
    ])

def fragment_outside_box(fragments,boxes):
    """
    if points in fragment outside box

    Calculate line distance among points of fragment i and lines of box j, get [P,B,4]
    if all 4 values of a point are great than or equal to 0, the point is inside the box
    else the point is in outside the box

    Parameters:
    ----------
    fragments: [F,FP,2]
    boxes: [B,4]

    Return:
    ----------
    ret: [F,FP,B]
    """
    assert fragments.dim()==3
    F,FP,_ = fragments.shape
    B,_ = boxes.shape

    diff = torch.cat([
        fragments.view(F,FP,1,2)-boxes[:,:2].view(1,1,B,2),
        boxes[:,2:].view(1,1,B,2)-fragments.view(F,FP,1,2)
    ],dim=-1)
    
    return ((diff>=0).sum(-1)!=4).float()

def fragment_box_distance(fragments,box_points):
    """
    calcuate distance among fragments of fragmaent_i and box_points of box_j
    get the smallest distance for each point of box_i to box_j

    Parameters:
    ----------
    fragments: [F,FP,2]
    box_points: [B,BP,2]

    Return:
    ----------
    ret: [F,FP,B]
    """    
    F,FP,_ = fragments.shape
    B,BP,_ = box_points.shape
    # [F,FP,B,BP] -> [F,FP,B]
    #return (fragments.view(F,FP,1,1,2)-box_points.view(1,1,B,BP,2)).norm(dim=-1).min(-1)[0]
    return (fragments.view(F,FP,1,1,2)-box_points.view(1,1,B,BP,2)).pow(2).sum(-1).min(-1)[0]

def coverage_loss(boxes,fragments,step=2):
    """
    boxes can cover all points in fragments

    Parameters:
    ----------
    boxes: [B,4], NBoxes with (x0,y0,x1,y1)
    fragments: [F,FP,2], NBox wit (x,y)
    """
    BP = step*4
    B, _ = boxes.shape
    F,FP, _ = fragments.shape
    box_wh=boxes[:,2:]-boxes[:,:2]

    # [B,BP,2]
    box_points = sample_boundary(step=step).view(1,BP,2)*box_wh.view(B,1,2)+boxes[:,:2].view(B,1,2)
    # [F,FP,B]
    f_out_box = fragment_outside_box(fragments,boxes)
    # [F,FP,B]
    f_b_dist = fragment_box_distance(fragments,box_points)
    # [F,FP]
    return (f_b_dist*f_out_box).min(-1)[0].sum()/FP

def inside_loss(boxes,fragment_boxes,step=2):
    """
    all points on the boundary of boxes are inside fragment_boxes

    Parameters:
    ----------
    boxes: [B,4], NBoxes with (x0,y0,x1,y1)
    fragment_boxes: [F,4], NBox wit (x,y)
    """
    B, _ = boxes.shape
    P = step*4 
    F, _ = fragment_boxes.shape
    box_wh=boxes[:,2:]-boxes[:,:2]
    fragment_wh = fragment_boxes[:,2:]-fragment_boxes[:,:2]

    # [B,BP,2]
    box_fragments = sample_boundary(step=step).view(1,P,2)*box_wh.view(B,1,2)+boxes[:,:2].view(B,1,2)
    # [F,FP,2]
    fragment_boundaries = sample_boundary(step=step).view(1,P,2)*fragment_wh.view(F,1,2)+fragment_boxes[:,:2].view(F,1,2)
    # [B,BP,F]
    f_out_box = fragment_outside_box(box_fragments,fragment_boxes)
    # [B,BP,F]
    f_b_dist = fragment_box_distance(box_fragments,fragment_boundaries)
    # [B,BP]
    return (f_b_dist*f_out_box).sum()/(B*P)

def mutex_loss(boxes,step=2):
    """
    sum of min-pixel-boundary distance / sum of pixels in boxes

    Parameters:
    ----------
    boxes: B*4, bboxes with (x0,y0,x1,y1)
    ref_points: P*2, bbox wit (x,y)
    """    
    B = boxes.shape[0]
    BP = step*4
    FP = step*step
    box_wh=boxes[:,2:]-boxes[:,:2]

    # [B,FP,2]
    fragments = sample_fragment(step=step).view(1,FP,2)*box_wh.view(B,1,2)+boxes[:,:2].view(B,1,2)
    # [B,BP,2]
    box_points = sample_boundary(step=step).view(1,BP,2)*box_wh.view(B,1,2)+boxes[:,:2].view(B,1,2)

    # [B,FP,B]
    f_out_box = fragment_outside_box(fragments,boxes)
    # [B,FP,B] B个Box的PP个点与B个Box的最小距离
    f_b_dist = fragment_box_distance(fragments,box_points)

    return (f_b_dist*f_out_box).sum()/(B*FP-B)

class InsideLoss(nn.Module):
    def __init__(self,nsample=100,cuda=True):
        super(InsideLoss,self).__init__()
        self.bstep = round(nsample/4)
        self.fstep = round(math.sqrt(nsample))
        self.BP = self.bstep*4
        self.FP = self.fstep*self.fstep
        self.boundary = sample_boundary(step=self.bstep).view(1,self.BP,2)
        self.fragment = sample_fragment(step=self.fstep).view(1,self.FP,2)
        
        if cuda:
            self.boundary=self.boundary.cuda()
            self.fragment=self.fragment.cuda()

    def _inside_loss(self,boxes,fragment_boxes):
        B, _ = boxes.shape
        F, _ = fragment_boxes.shape
        box_wh=boxes[:,2:]-boxes[:,:2]
        fragment_wh = fragment_boxes[:,2:]-fragment_boxes[:,:2]

        # [B,FP,2]
        box_fragments = self.fragment*box_wh.view(B,1,2)+boxes[:,:2].view(B,1,2)
        # [F,BP,2]
        fragment_boundaries = self.boundary*fragment_wh.view(F,1,2)+fragment_boxes[:,:2].view(F,1,2)
        # [B,FP,F]
        f_out_box = fragment_outside_box(box_fragments,fragment_boxes)
        # [B,FP,F]
        f_b_dist = fragment_box_distance(box_fragments,fragment_boundaries)
        # [B,FP]
        return (f_b_dist*f_out_box).sum()/(B*self.FP)   

    def test(self,boxes,fragment_boxes):
        with torch.no_grad():
            return self._inside_loss(boxes,fragment_boxes)

    def forward(self,boxes,fragment_boxes,obj_to_img,reduction="mean"):
        N = obj_to_img.data.max().item() + 1
        boxes = box_utils.centers_to_extents(boxes)

        losses = []
        for i in range(N):
            obj_to_i = (obj_to_img==i).nonzero().view(-1)
            loss = self._inside_loss(boxes[obj_to_i],fragment_boxes[[i]])
            losses.append(loss)
        return torch.mean(torch.stack(losses))

class CoverageLoss(nn.Module):
    def __init__(self,nsample=100,cuda=True):
        super(CoverageLoss,self).__init__()
        self.step = round(nsample/4)
        self.BP = self.step*4
        self.boundary = sample_boundary(step=self.step).view(1,self.BP,2)
        self.boundary = self.boundary.cuda()

    def _coverage_loss(self,boxes,fragments):
        B, _ = boxes.shape
        F,FP, _ = fragments.shape
        box_wh=boxes[:,2:]-boxes[:,:2]

        # [B,BP,2]
        box_points = self.boundary*box_wh.view(B,1,2)+boxes[:,:2].view(B,1,2)
        # [F,FP,B]
        f_out_box = fragment_outside_box(fragments,boxes)
        # [F,FP,B]
        f_b_dist = fragment_box_distance(fragments,box_points)
        # [F,FP]
        return (f_b_dist*f_out_box).min(-1)[0].sum()/FP   

    def test(self,boxes,fragments):
        with torch.no_grad():
            return self._coverage_loss(boxes,fragments)

    def forward(self,boxes,fragments,obj_to_img,reduction="mean"):
        N = obj_to_img.data.max().item() + 1
        boxes = box_utils.centers_to_extents(boxes)

        losses = []
        for i in range(N):
            obj_to_i = (obj_to_img==i).nonzero().view(-1)
            loss = self._coverage_loss(boxes[obj_to_i],fragments[i])
            losses.append(loss)
        return torch.mean(torch.stack(losses))

class MutexLoss(nn.Module):
    def __init__(self,nsample=100,cuda=True):
        super(MutexLoss,self).__init__()
        self.bstep = round(nsample/4)
        self.fstep = round(math.sqrt(nsample))
        self.BP = self.bstep*4
        self.FP = self.fstep*self.fstep
        self.boundary = sample_boundary(step=self.bstep).view(1,self.BP,2)
        self.fragment = sample_fragment(step=self.fstep).view(1,self.FP,2)
        if cuda:
            self.boundary=self.boundary.cuda()
            self.fragment=self.fragment.cuda()

    def _mutex_loss(self,boxes):
        B = boxes.shape[0]
        box_wh=boxes[:,2:]-boxes[:,:2]

        # [B,FP,2]
        fragments = self.fragment*box_wh.view(B,1,2)+boxes[:,:2].view(B,1,2)
        # [B,BP,2]
        box_points = self.boundary*box_wh.view(B,1,2)+boxes[:,:2].view(B,1,2)

        # [B,FP,B]
        f_in_box = 1-fragment_outside_box(fragments,boxes)
        # clear dist to self
        f_in_box[range(B),:,range(B)]=0
        # [B,FP,B] B个Box的PP个点与B个Box的最小距离
        f_b_dist = fragment_box_distance(fragments,box_points)

        return (f_b_dist*f_in_box).sum()/(B*self.FP-B)     
    
    def test(self,boxes):
        with torch.no_grad():
            return self._mutex_loss(boxes)
    
    def forward(self,boxes,obj_to_img,objs=None,reduction="mean"):
        N = obj_to_img.data.max().item() + 1
        boxes = box_utils.centers_to_extents(boxes)
        losses = []
        for i in range(N):
            obj_to_i = ((obj_to_img==i)*(objs!=0)).nonzero().view(-1)
            loss = self._mutex_loss(boxes[obj_to_i])
            losses.append(loss)
        return torch.mean(torch.stack(losses))

class BoxRenderLoss(nn.Module):
    def __init__(self,nsample=100,cuda=True):
        super(BoxRenderLoss,self).__init__()
        self.bstep = round(nsample/4)
        self.fstep = round(math.sqrt(nsample))
        self.BP = self.bstep*4
        self.FP = self.fstep*self.fstep
        self.boundary = sample_boundary(step=self.bstep).view(1,self.BP,2)
        self.fragment = sample_fragment(step=self.fstep).view(1,self.FP,2)
        if cuda:
            self.boundary=self.boundary.cuda()
            self.fragment=self.fragment.cuda()

    def _fragment_outside_box(self,fragments,boxes):
        assert fragments.dim()==3
        F,FP,_ = fragments.shape

        diff = torch.cat([
            fragments.view(F,FP,2)-boxes[:,:2].view(F,1,2),
            boxes[:,2:].view(F,1,2)-fragments.view(F,FP,2)
        ],dim=-1)
        
        return ((diff>=0).sum(-1)!=4).float()

    def _fragment_box_distance(self,fragments,box_points):
        F,FP,_ = fragments.shape
        B,BP,_ = box_points.shape
        return (fragments.view(F,FP,1,2)-box_points.view(B,1,BP,2)).pow(2).sum(-1).min(-1)[0]

    def _render_loss(self,boxes,targets):
        B = boxes.shape[0]
        box_wh=boxes[:,2:]-boxes[:,:2]
        target_wh=targets[:,2:]-targets[:,:2]

        # [B,FP,2]
        box_fragments = self.fragment*box_wh.view(B,1,2)+boxes[:,:2].view(B,1,2)
        # [B,BP,2]
        box_points = self.boundary*box_wh.view(B,1,2)+boxes[:,:2].view(B,1,2)

        # [B,FP,2]
        target_fragments = self.fragment*target_wh.view(B,1,2)+targets[:,:2].view(B,1,2)
        # [B,BP,2]
        target_points = self.boundary*target_wh.view(B,1,2)+targets[:,:2].view(B,1,2)

        # [B,FP]
        b_out_t = self._fragment_outside_box(box_fragments,targets)
        t_out_b = self._fragment_outside_box(target_fragments,boxes)

        # [B,FP] B个Box的PP个点与B个Box的最小距离
        b_t_dist = self._fragment_box_distance(box_fragments,target_points)
        t_b_dist = self._fragment_box_distance(target_fragments,box_points)

        return ((b_t_dist*b_out_t).sum()+(t_b_dist*t_out_b).sum())/(2*B*self.FP)

    def test(self,boxes,targets):
        with torch.no_grad():
            return _render_loss(self,boxes,targets)

    def forward(self,boxes,targets,reduction="mean"):
        return torch.mean(self._render_loss(boxes,targets))


class DoorLoss(nn.Module):
    def __init__(self,nsample=100,cuda=True):
        super(DoorLoss,self).__init__()
        self.bstep = round(nsample/4)
        self.fstep = round(math.sqrt(nsample))
        self.BP = self.bstep*4
        self.FP = self.fstep*self.fstep
        self.boundary = sample_boundary(step=self.bstep).view(1,self.BP,2)
        self.fragment = sample_fragment(step=self.fstep).view(1,self.FP,2)
        if cuda:
            self.boundary=self.boundary.cuda()
            self.fragment=self.fragment.cuda()

    def _door_loss(self,boxes,doors,objs):
        B, _ = boxes.shape
        F, _ = doors.shape
        box_wh=boxes[:,2:]-boxes[:,:2]
        door_wh = doors[:,2:]-doors[:,:2]

        # [B,BP,2]
        box_boundaries = self.boundary*box_wh.view(B,1,2)+boxes[:,:2].view(B,1,2)
        # [F,FP,2]
        door_fragments = self.fragment*door_wh.view(F,1,2)+doors[:,:2].view(F,1,2)
        # [F,FP,B]
        f_out_box = (objs-fragment_outside_box(door_fragments,boxes)).abs()
        # [F,FP,B]
        f_b_dist = fragment_box_distance(door_fragments,box_boundaries)
        # [F,FP]
        return (f_b_dist*f_out_box).sum()/(F*self.FP) 

    def forward(self,boxes,doors,obj_to_img,objs=None,reduction="mean"):
        N = obj_to_img.data.max().item() + 1
        boxes = box_utils.centers_to_extents(boxes)

        losses = []
        for i in range(N):
            obj_to_i = ((obj_to_img==i)).nonzero().view(-1)
            objs_i = (objs[obj_to_img==i]!=0).long()
            loss = self._door_loss(boxes[obj_to_i],doors[[i]],objs_i)
            losses.append(loss)
        return torch.mean(torch.stack(losses))

if __name__ == "__main__":
    # [4]
    fragment_box = torch.tensor(
        [0.25,0.25,0.75,0.75]
    ).view(1,4)
    # [B,4]
    boxes = torch.tensor([
        [0.25,0.00,0.50,0.50],
        [0.50,0.50,1.00,0.75]
    ],requires_grad=True)
    # (0.25**2*2+(0.25*sqrt(2))**2*2)/(8*2)=0.046875
    loss = inside_loss(boxes,fragment_box,step=2)
    print(loss)
    insideL = InsideLoss(cuda=False)
    print(insideL(boxes,fragment_box,torch.tensor([0,0])))

    boxes = torch.tensor([
        [0.25,0.00,0.50,0.50],
        [0.50,0.50,1.00,0.75]
    ],requires_grad=True)
    # 
    X = torch.linspace(0.,1.,4)
    Y = torch.linspace(0.,1.,4)
    # P,2
    fragment_points = torch.tensor([ (X[x],Y[y]) for x in range(1,3) for y in range(1,3)]).view(1,4,2)
    # (0.09**2+0.16**2+0.16**2*2)/4=0.021225
    loss = coverage_loss(boxes,fragment_points)
    print(loss)
    coverageL = CoverageLoss(cuda=False)
    print(coverageL(boxes,fragment_points,torch.tensor([0,0])))

    # B,4
    boxes = torch.tensor([
        [0.25,0.00,0.49,0.49], #0
        [0.00,0.25,0.49,0.49], #1
        [0.51,0.51,1.00,0.75], #2
        [0.51,0.51,0.75,1.00]  #3
    ],requires_grad=True)

    # (( ( (0.24**2+0.25**2) + (0.51**2+0.26**2)*2 )
    # +(0.25**2+ (0.51**2+0.02**2)*2 )
    # +((0.26**2+0.02**2)*2)
    # +((0.02**2*2)*2))*4)
    # /(4*4-4) = 0.4988666
    loss = mutex_loss(boxes,step=2)
    print(loss)
    mutexL = MutexLoss(cuda=False)
    print(mutexL(boxes,torch.tensor([0,0,1,1])))