Network/model/floorplan.py

import torch
from torch.utils.data import Dataset
import numpy as np
import scipy.io as sio
import cv2
import copy
from model.utils import *

class FloorPlan():

    def __init__(self, data, rot=None,fliplr=False):
        self.data = copy.deepcopy(data)

        ''' transform '''
        if rot is not None:
            theta = self._get_rot()
            self.data.gtBoxNew = align_box(self.data.gtBoxNew[:,[1,0,3,2]],theta,rot)[:,[1,0,3,2]]
            self.data.boundary[:,[1,0]] = align_points(self.data.boundary[:,[1,0]],theta,rot)
        
        if fliplr:
            self.data.gtBoxNew[:,[1,0,3,2]] = fliplr_box(self.data.gtBoxNew[:,[1,0,3,2]])
            self.data.boundary[:,[1,0]] = fliplr_2D(self.data.boundary[:,[1,0]])

    def _get_rot(self):
        door_line = self.data.boundary[:2, :2]  # [:,[1,0]]
        c = door_line.mean(0) - np.array([127.5,127.5])
        theta = np.arctan2(c[1], c[0]) + np.pi  # [-pi,pi]
        return theta

    def get_input_boundary(self, tensor=True):
        external = self.data.boundary[:, :2]
        door = self.data.boundary[:2, :2]

        boundary = np.zeros((128, 128), dtype=float)
        inside = np.zeros((128, 128), dtype=float)
        front = np.zeros((128, 128), dtype=float)

        pts = np.concatenate([external, external[:1]]) // 2
        pts_door = door // 2

        cv2.fillPoly(inside, pts.reshape(1, -1, 2), 1.0)
        cv2.polylines(boundary, pts.reshape(1, -1, 2), True, 1.0, 3)
        cv2.polylines(boundary, pts_door.reshape(1, -1, 2), True, 0.5, 3)
        cv2.polylines(front, pts_door.reshape(1, -1, 2), True, 1.0, 3)

        input_image = np.stack([inside, boundary, front], -1)
        if tensor: input_image = torch.tensor(input_image).permute((2, 0, 1)).float()
        return input_image

    def get_inside_box(self, tensor=True):
        boundary = self.data.boundary[:, :2]

        X, Y = np.linspace(0, 1, 256), np.linspace(0, 1, 256)
        x0, x1 = np.min(boundary[:, 0]), np.max(boundary[:, 0])
        y0, y1 = np.min(boundary[:, 1]), np.max(boundary[:, 1])
        box = np.array([[X[x0], Y[y0], X[x1], Y[y1]]])
        if tensor: box = torch.tensor(box).float()
        return box

    def get_rooms(self, tensor=True):
        rooms = self.data.rType
        if tensor: rooms = torch.tensor(rooms).long()
        return rooms

    def get_attributes(self, gsize=5, alevel=10, relative=True, tensor=True):
        boxes = self.data.gtBoxNew[:,[1,0,3,2]]
        boundary = self.data.boundary[:,:2]

        h, w = 256, 256
        if relative:
            x0, x1 = np.min(boundary[:, 0]), np.max(boundary[:, 0])+1
            y0, y1 = np.min(boundary[:, 1]), np.max(boundary[:, 1])+1
            h, w = y1 - y0, x1 - x0
            boxes = boxes - np.array([y0, x0, y0, x0], dtype=float)

        boxes /= np.array([h, w, h, w])
        boxes[:, 2:] -= boxes[:, :2]  # y1,x1->h,w
        boxes[:, :2] += boxes[:, 2:] / 2  # y0,x0->yc,xc
        
        l = len(boxes)
        gbins = np.linspace(0,1,gsize+1) # [1,gsize]
        gbins[0],gbins[-1]=-np.inf,np.inf
        abins = np.linspace(0,1,alevel+1) # [1,gsize]
        abins[0],abins[-1]=-np.inf,np.inf
        
        attributes = np.zeros((l,gsize*gsize+alevel))
        # pos: xc*gsize+yc*gsize*gsize
        attributes[range(l),(np.digitize(boxes[:,0],gbins)-1)*gsize+np.digitize(boxes[:,1],gbins)-1]=1
        # area:(w*h)
        attributes[range(l),gsize*gsize+np.digitize(boxes[:,2:].prod(1),abins)-1]=1
        if tensor: attributes = torch.tensor(attributes).float()
        return attributes

    def get_triples(self, random=False, tensor=True):
        boxes = self.data.gtBoxNew[:, [1, 0, 3, 2]]
        vocab = get_vocab()

        triples = []
        # add edge relation
        for u, v, _ in self.data.rEdge:
            # Todo: random order for edge
            # if random and np.random.random() > 0.5:
            #     u, v = v, u
            uy0, ux0, uy1, ux1 = boxes[u]
            vy0, vx0, vy1, vx1 = boxes[v]
            uc = (uy0 + uy1) / 2, (ux0 + ux1) / 2
            vc = (vy0 + vy1) / 2, (vx0 + vx1) / 2

            # surrounding/inside -> X four quadrants
            if ux0 < vx0 and ux1 > vx1 and uy0 < vy0 and uy1 > vy1:
                relation = 'surrounding'
            elif ux0 >= vx0 and ux1 <= vx1 and uy0 >= vy0 and uy1 <= vy1:
                relation = 'inside'
            else:
                relation = point_box_relation(uc, boxes[v])

            triples.append([u, vocab['pred_name_to_idx'][relation], v])

        triples = np.array(triples, dtype=int)
        if tensor: triples = torch.tensor(triples).long()
        return triples

    def get_layout_image(self,tensor=True):
        img = np.full((128,128),13,dtype=np.uint8)
        boundary = self.data.boundary[:,:2]
        boundary = np.concatenate([boundary, boundary[:1]])

        cv2.fillPoly(img, (boundary//2).reshape(1, -1, 2), 0)

        order = self.data.order-1
        rType = self.data.rType[order]
        rBox = self.data.gtBoxNew[order]

        for i in range(len(rType)):
            t = rType[i]
            if t==0: continue
            b = rBox[i]//2
            img[b[1]:b[3],b[0]:b[2]]=t
        
        cv2.polylines(img, (boundary//2).reshape(1, -1, 2), True,14)

        if tensor: img = torch.tensor(img).long()
        return img

    def get_boxes(self,relative=True,tensor=True):
        boxes = self.data.gtBoxNew[:, [1, 0, 3, 2]]
        boundary = self.data.boundary[:,:2]
        
        X,Y = np.linspace(0,1,256),np.linspace(0,1,256)

        if relative:
            x0, x1 = np.min(boundary[:, 0]), np.max(boundary[:, 0])+1
            y0, y1 = np.min(boundary[:, 1]), np.max(boundary[:, 1])+1
            h, w = y1 - y0, x1 - x0
            X,Y = np.linspace(0,1,w),np.linspace(0,1,h)
            boxes = boxes-np.array([y0,x0,y0,x0])

        norm = lambda box:np.array([
            X[max(box[1],0)],
            Y[max(box[0],0)],
            X[min(box[3]-1,w-1)],
            Y[min(box[2]-1,h-1)]
        ])
        
        boxes = np.apply_along_axis(norm,1,boxes)
        boxes[:,2:]-=boxes[:,:2]
        boxes[:,:2]+=boxes[:,2:]/2
        if tensor: 
            boxes = torch.tensor(boxes).float()
        return boxes

    def get_inside_coords(self,size=(32,32),tensor=True):
        h,w = size
        X = np.linspace(0,1,w)
        Y = np.linspace(0,1,h)
        img = np.zeros(size)
        boundary = self.data.boundary[:,:2]
        boundary = boundary*np.array(size)//256

        cv2.fillPoly(img, boundary.reshape(1, -1, 2), 1)

        coords = np.where(img>0)
        coords = np.stack((X[coords[1]],Y[coords[0]]),1)
        if tensor: coords = torch.tensor(coords).unsqueeze(0).float()
        return coords

    def get_test_data(self, tensor=True):
        name = self.data.name

        boundary = self.get_input_boundary(tensor=tensor)
        inside_box = self.get_inside_box(tensor=tensor)
        rooms = self.get_rooms(tensor=tensor)
        attrs = self.get_attributes(tensor=tensor)
        triples = self.get_triples(random=False, tensor=tensor)
        return boundary, inside_box, rooms, attrs, triples, name

    def get_train_data(self, tensor=True):
        name = self.data.name

        boundary = self.get_input_boundary(tensor=tensor)
        inside_box = self.get_inside_box(tensor=tensor)
        rooms = self.get_rooms(tensor=tensor)
        attrs = self.get_attributes(tensor=tensor)
        triples = self.get_triples(random=False, tensor=tensor)

        # gt
        layout = self.get_layout_image(tensor=tensor)
        boxes = self.get_boxes(tensor=tensor)
        
        # constrains
        inside_coords = self.get_inside_coords(tensor=tensor)
        
        return boundary,inside_box,rooms,attrs,triples,layout,boxes,inside_coords,name

class FloorPlanDataset(Dataset):
    def __init__(self,data_path):
        self.data = sio.loadmat(data_path, squeeze_me=True, struct_as_record=False)['data']

        self.train = True if 'train' in data_path else False
    
    def __len__(self):
        return len(self.data)
    
    def __getitem__(self,i):
        if self.train:
            rot = np.random.randint(0,4)
            fliplr = np.random.random()>0.5
            fp = FloorPlan(self.data[i],rot=rot,fliplr=fliplr)
            return fp.get_train_data()
        else:
            fp = FloorPlan(self.data[i])
            return fp.get_train_data()
    
def floorplan_collate_fn(batch):
    all_boundary = []
    all_inside_box = []
    all_objs = []
    all_attrs = []
    all_triples = []

    all_layout = []
    all_boxes = []

    all_inside_coords = []
    
    all_obj_to_img = []
    all_triple_to_img = []

    all_name = []

    obj_offset = 0
    for i, (
        boundary,
        inside_box,
        rooms,
        attrs,
        triples,
        layout,
        boxes,
        inside_coords,
        name
        ) in enumerate(batch):
        if rooms.dim() == 0 or triples.dim() == 0:
            continue
        O, T = rooms.size(0), triples.size(0)

        all_boundary.append(boundary[None])
        all_inside_box.append(inside_box)

        all_objs.append(rooms)
        all_attrs.append(attrs)
        triples = triples.clone()
        triples[:, 0] += obj_offset
        triples[:, 2] += obj_offset
        all_triples.append(triples)

        all_layout.append(layout[None])
        all_boxes.append(boxes)
        all_inside_coords.append(inside_coords)

        all_name.append(name)
        
        all_obj_to_img.append(torch.LongTensor(O).fill_(i))
        all_triple_to_img.append(torch.LongTensor(T).fill_(i))

        obj_offset += O

    all_boundary = torch.cat(all_boundary)
    all_inside_box = torch.cat(all_inside_box)

    all_objs = torch.cat(all_objs)
    all_attrs = torch.cat(all_attrs)
    all_triples = torch.cat(all_triples)

    all_layout = torch.cat(all_layout)
    all_boxes = torch.cat(all_boxes)

    all_obj_to_img = torch.cat(all_obj_to_img)
    all_triple_to_img = torch.cat(all_triple_to_img)

    out = (
        all_boundary,
        all_inside_box, 
        all_objs,
        all_attrs, 
        all_triples,

        all_layout, 
        all_boxes, 

        all_inside_coords, 
        
        all_obj_to_img,
        all_triple_to_img,
        all_name
    )
    return out
    

def vis_fp(fp,size=(256,256)):
    cmap = get_color_map()

    img = np.full((256, 256, 4), 0, dtype=np.uint8)

    boundary = fp.data.boundary[:,:2]
    boundary = np.concatenate([boundary, boundary[:1]])
    door = fp.data.boundary[:2, :2]

    c = cmap[0].tolist()
    cv2.fillPoly(img, boundary.reshape(1, -1, 2), (c[0],c[1],c[2],255))

    order = fp.data.order-1
    rType = fp.data.rType[order]
    rBox = fp.data.gtBoxNew[order]

    for i in range(len(rType)):
        t = rType[i]
        if t==0: continue
        b = rBox[i]
        c = cmap[t].tolist()
        cv2.rectangle(img, (b[0],b[1]),(b[2]-1,b[3]-1), (c[0],c[1],c[2],255),-1)
        c = cmap[-1].tolist()
        cv2.rectangle(img, (b[0],b[1]),(b[2]-1,b[3]-1), (c[0],c[1],c[2],255),3)
        cv2.putText(img,f"{i}",((b[0]+b[2])//2,(b[1]+b[3])//2),cv2.FONT_HERSHEY_PLAIN,1,(0,0,0,255))

    c = cmap[-3].tolist()
    cv2.polylines(img, boundary.reshape(1, -1, 2), True, (c[0],c[1],c[2],255),3)
    
    c = cmap[-2].tolist()
    cv2.polylines(img, door.reshape(1, -1, 2), True, (c[0],c[1],c[2],255),3)

    if size!=(256,256): return cv2.resize(img,size)
    return img

if __name__ == "__main__":
    pass