MoFlow/data/dataloader_nba.py

import os
import matplotlib.pyplot as plt
import numpy as np
import pickle
import glob
from torch.utils.data import Dataset
from utils.normalization import normalize_min_max, unnormalize_min_max, normalize_sqrt, unnormalize_sqrt
import torch
from utils.utils import rotate_trajs_x_direction


def seq_collate_nba(batch):
    (past_traj, fut_traj, past_traj_orig, fut_traj_orig, traj_vel) = zip(*batch)
    pre_motion_3D = torch.stack(past_traj,dim=0)
    fut_motion_3D = torch.stack(fut_traj,dim=0)
    pre_motion_3D_orig = torch.stack(past_traj_orig, dim=0)
    fut_motion_3D_orig = torch.stack(fut_traj_orig, dim=0)
    fut_traj_vel = torch.stack(traj_vel, dim=0)

    batch_size = torch.tensor(pre_motion_3D.shape[0]) ### bt 
    traj_mask = torch.zeros(batch_size * 11, batch_size * 11)
    for i in range(batch_size):
        traj_mask[i*11:(i+1)*11, i*11:(i+1)*11] = 1.
    data = {
        'batch_size': batch_size,
        'past_traj': pre_motion_3D,
        'fut_traj': fut_motion_3D,
        'past_traj_original_scale': pre_motion_3D_orig,
        'fut_traj_original_scale': fut_motion_3D_orig,
        'traj_mask': traj_mask,
        'fut_traj_vel': fut_traj_vel,  
    }

    return data 

def seq_collate_imle_train(batch):
    (past_traj, fut_traj, past_traj_orig, fut_traj_orig, traj_vel, y_t, y_pred_data) = zip(*batch)

    pre_motion_3D = torch.stack(past_traj,dim=0)
    fut_motion_3D = torch.stack(fut_traj,dim=0)
    pre_motion_3D_orig = torch.stack(past_traj_orig, dim=0)
    fut_motion_3D_orig = torch.stack(fut_traj_orig, dim=0)
    fut_traj_vel = torch.stack(traj_vel, dim=0)
    y_t = torch.stack(y_t, dim=0)
    y_pred_data = torch.stack(y_pred_data,dim=0)

    batch_size = torch.tensor(pre_motion_3D.shape[0]) ### bt 
    traj_mask = torch.zeros(batch_size * 11, batch_size * 11)
    for i in range(batch_size):
        traj_mask[i*11:(i+1)*11, i*11:(i+1)*11] = 1.
    data = {
        'batch_size': batch_size,
        'past_traj': pre_motion_3D,
        'fut_traj': fut_motion_3D,
        'past_traj_original_scale': pre_motion_3D_orig,
        'fut_traj_original_scale': fut_motion_3D_orig,
        'fut_traj_vel': fut_traj_vel,
        'traj_mask': traj_mask,
        'y_t': y_t,
        'y_pred_data': y_pred_data
    }

    return data


class NBADatasetMinMax(Dataset):
    """Dataloder for the Trajectory datasets"""
    def __init__(
        self,
        obs_len=5, 
        pred_len=10, 
        training=True, 
        num_scenes=32500, 
        test_scenes=12500,
        overfit=False, 
        traj_scale_total=94/28,
        imle=False,
        cfg=None,
        data_dir='/data/nba',
        rotate=False,
        data_norm='min_max'
    ):
        """
        Args:
        - obs_len: Number of time-steps in input trajectories
        - pred_len: Number of time-steps in output trajectories
        - skip: Number of frames to skip while making the dataset
        - imle: Whether we train with IMLE or not (a switch)
        """

        super(NBADatasetMinMax, self).__init__()

        self.obs_len = obs_len
        self.pred_len = pred_len
        self.seq_len = self.obs_len + self.pred_len
        self.imle = imle 
        self.traj_mean = torch.FloatTensor(cfg.traj_mean).unsqueeze(0).unsqueeze(0).unsqueeze(0)

        if not overfit:
            if training:
                data_root = os.path.join(data_dir, 'original/nba_train.npy')
            else:
                data_root = os.path.join(data_dir, 'original/nba_test.npy')
        else:
            data_root = os.path.join(data_dir, 'original/nba_train.npy')

        self.trajs_raw = np.load(data_root) #(N,15,11,2)
        self.trajs = self.trajs_raw / traj_scale_total
        if training:
            self.trajs = self.trajs[:num_scenes]
        else:
            self.trajs = self.trajs[:test_scenes]

        ### Overfit test 
        if overfit:
            self.trajs = self.trajs[:num_scenes]
        

        self.data_len = len(self.trajs)
        print("Size of the dataset: {} in {} mode".format(self.data_len, 'Training' if training else 'Testing'))

        self.traj_abs = torch.from_numpy(self.trajs).type(torch.float)

        self.traj_abs = self.traj_abs.permute(0,2,1,3)
        self.actor_num = self.traj_abs.shape[1]

        pre_motion_3D = self.traj_abs[:, :, :self.obs_len, :]
        fut_motion_3D = self.traj_abs[:, :, self.obs_len:, :]
        initial_pos = pre_motion_3D[:, :, -1:]

        # augment input: absolute position, relative position, velocity
        fut_traj = (fut_motion_3D - initial_pos).contiguous()
        past_traj_abs = (pre_motion_3D - self.traj_mean).contiguous()
        past_traj_rel = (pre_motion_3D - initial_pos).contiguous() 
        if rotate:
            past_traj_rel, fut_traj, past_traj_abs = rotate_trajs_x_direction(past_traj_rel, fut_traj, past_traj_abs)
        past_traj_vel = torch.cat((past_traj_rel[:, :, 1:] - past_traj_rel[:, :, :-1], torch.zeros_like(past_traj_rel[:, :, -1:])), dim=2) 
        past_traj = torch.cat((past_traj_abs, past_traj_rel, past_traj_vel), dim=-1)
        self.fut_traj_vel = torch.cat((fut_traj[:, :, 1:] - fut_traj[:, :, :-1], torch.zeros_like(fut_traj[:, :, -1:])), dim=2)
        
        if training:
            cfg.fut_traj_max = fut_traj.max()
            cfg.fut_traj_min = fut_traj.min()
            cfg.past_traj_max = past_traj.max()
            cfg.past_traj_min = past_traj.min()
        
        ### record the original to avoid numerical errors
        self.past_traj_original_scale = past_traj
        self.fut_traj_original_scale = fut_traj
        
        self.data_norm = data_norm
        if data_norm == 'min_max':
            ### min-max linear normalization
            self.past_traj = normalize_min_max(past_traj, cfg.past_traj_min, cfg.past_traj_max, -1, 1).contiguous()
            self.fut_traj = normalize_min_max(fut_traj, cfg.fut_traj_min, cfg.fut_traj_max, -1, 1).contiguous()
        elif data_norm == 'sqrt':
            ### sqrt normalization
            sqrt_a_ = torch.tensor([cfg.sqrt_x_a, cfg.sqrt_y_a], device=past_traj.device)
            sqrt_b_ = torch.tensor([cfg.sqrt_x_b, cfg.sqrt_y_b], device=past_traj.device)

            # no need to normalize the past trajectory
            self.past_traj = past_traj


            self.fut_traj = normalize_sqrt(fut_traj, sqrt_a_, sqrt_b_).contiguous()

            # unnormalized_xy = unnormalize_sqrt(self.fut_traj, sqrt_a_, sqrt_b_)
            # unnorm_error = torch.abs(unnormalized_xy - fut_traj).mean()

        """load distillation target"""
        if imle:
            os.makedirs(os.path.join(data_dir, 'imle'), exist_ok=True)
            pkl_ls = sorted(glob.glob(os.path.join(data_dir, 'imle/*train*.pkl')))

            keys_ls = ['past_traj', 'fut_traj', 'past_traj_original_scale', 'fut_traj_original_scale', 'fut_traj_vel', 'y_t', 'y_pred_data']
            imle_data_dict = {}
            total_scenes_loaded_ = 0
            for i_pkl, cur_pkl in enumerate(pkl_ls):
                data = pickle.load(open(cur_pkl, 'rb'))

                if i_pkl == 0:
                    self.imle_meta_data = data['meta_data']
                
                for key in keys_ls:
                    if key not in imle_data_dict:
                        imle_data_dict[key] = []
                    if key == 'y_t':
                        imle_data_dict[key].append(data[key][:, -1])
                    else:
                        imle_data_dict[key].append(data[key])

                total_scenes_loaded_ += data['past_traj'].shape[0]

                if total_scenes_loaded_ >= len(self.trajs):
                    break

                # y_t_original_scale_ = unnormalize_min_max(torch.from_numpy(data['y_t'][:, -1]), cfg.fut_traj_min, cfg.fut_traj_max, -1, 1)
                # y_pred_data_original_scale_ = torch.from_numpy(data['y_pred_data'])
                # assert torch.sum(torch.abs(y_t_original_scale_ - y_pred_data_original_scale_)) < 1e-5, 'IMLE data is not consistent'
                        
                # past_tarj_original_scale_ = torch.from_numpy(data['past_traj_original_scale'])
                # assert torch.sum(torch.abs(past_tarj_original_scale_[:10] - self.past_traj_original_scale[:10])) < 1e-5, 'IMLE data is not consistent'

            # concat the data
            for key in keys_ls:
                imle_data_dict[key] = torch.from_numpy(np.concatenate(imle_data_dict[key], axis=0))[:len(self.trajs)]

            self.imle_data_dict = imle_data_dict


    def __len__(self):
        return self.data_len

    def __getitem__(self, index):
        if self.imle:
            out = [
                    self.imle_data_dict['past_traj'][index], 
                    self.imle_data_dict['fut_traj'][index],
                    self.imle_data_dict['past_traj_original_scale'][index],
                    self.imle_data_dict['fut_traj_original_scale'][index],
                    self.imle_data_dict['fut_traj_vel'][index],
                    self.imle_data_dict['y_t'][index],
                    self.imle_data_dict['y_pred_data'][index]
                ]
        else:
            out = [
                    self.past_traj[index], 
                    self.fut_traj[index],
                    self.past_traj_original_scale[index],
                    self.fut_traj_original_scale[index], 
                    self.fut_traj_vel[index]
                ]
        return out