import json import math import os from os.path import join as osp from typing import Any, Dict, List, Optional, Tuple, Union import random import numpy as np import torch from lightning import LightningDataModule from omegaconf import DictConfig, ListConfig from PIL import Image from torch.utils.data import DataLoader, Dataset class MultiViewDataset(Dataset): def __init__( self, type_of_dataset: str, root_dir: str, num_views: int, num_derived_views: int, bg_color: str, img_wh: Tuple[int, int], caption_path: str, ref_view_mode: Union[str, List] = "random", sample_views_mode: Union[str, List] = "random", sample_derived_views_mode: Union[str, List] = "random", k_near_views: Optional[int] = None, max_samples_per_category: Optional[int] = None, use_abs_elevation: bool = False, seed = 0 ): #data = [] # # Loop per ogni cartella di categoria # for category in os.listdir(root_dir): # category_path = os.path.join(root_dir, category) # if os.path.isdir(category_path): # uids = os.listdir(category_path) # #se validazione, prendiamo per semplicità solo due campioni # if max_samples_per_category is not None: # random.seed(seed) # uids = random.sample(uids, min(max_samples_per_category, len(uids))) # for uid in uids: # uid_path = os.path.join(root_dir, category, uid) # obj_data = {"obj_dir": uid_path, "caption": ""} # data.append(obj_data) data = [] # Loop per ogni cartella di categoria for category in os.listdir(root_dir): category_path = os.path.join(root_dir, category) if os.path.isdir(category_path): uids = [f for f in os.listdir(category_path) if not os.path.isfile(os.path.join(category_path, f))] #os.listdir(category_path) if len(uids)!=0: for uid in uids: uid_path = os.path.join(root_dir, category, uid) obj_data = {"obj_dir": uid_path, "caption": ""} data.append(obj_data) else: uid_path = os.path.join(root_dir, category) obj_data = {"obj_dir": uid_path, "caption": ""} data.append(obj_data) #da eliminare (per velocità prendo solo i primi 8) if type_of_dataset=="validation": data = data[:144] self.data = data self.num_views = num_views self.num_derived_views = num_derived_views self.bg_color = bg_color self.img_wh = img_wh self.ref_view_mode = ref_view_mode self.sample_views_mode = sample_views_mode self.sample_derived_views_mode = sample_derived_views_mode self.k_near_views = k_near_views if k_near_views else num_views self.use_abs_elevation = use_abs_elevation def get_bg_color(self, bg_color): if bg_color == "white": bg_color = np.array([1.0, 1.0, 1.0], dtype=np.float32) elif bg_color == "black": bg_color = np.array([0.0, 0.0, 0.0], dtype=np.float32) elif bg_color == "gray": bg_color = np.array([0.5, 0.5, 0.5], dtype=np.float32) elif bg_color == "random": bg_color = np.random.rand(3) elif bg_color == "random_gray": bg_color = np.random.rand(1) * 0.5 + 0.3 bg_color = np.array([bg_color] * 3, dtype=np.float32) elif isinstance(bg_color, float): bg_color = np.array([bg_color] * 3, dtype=np.float32) else: raise NotImplementedError return bg_color def load_image(self, img_path, bg_color, rescale=True, return_type="np"): # not using cv2 as may load in uint16 format # img = cv2.imread(img_path, cv2.IMREAD_UNCHANGED) # [0, 255] # img = cv2.resize(img, self.img_wh, interpolation=cv2.INTER_CUBIC) # pil always returns uint8 img = np.array(Image.open(img_path).resize(self.img_wh)) img = img.astype(np.float32) / 255.0 # [0, 1] if img.shape[-1] == 4: alpha = img[..., 3:4] img = img[..., :3] * alpha + bg_color * (1 - alpha) if rescale: img = img * 2.0 - 1.0 # to -1 ~ 1 if return_type == "np": pass elif return_type == "pt": img = torch.from_numpy(img) else: raise NotImplementedError return img def get_k_near_anchors(self, elevations, azimuths, k_near_views, num_views, exclude_ids=None): views = torch.cat((elevations.unsqueeze(1), azimuths.unsqueeze(1)), dim=1) distances = torch.cdist(views, views) # Imposta la diagonale a inf per non contare sé stessi distances.fill_diagonal_(float('inf')) if exclude_ids is not None: # Esclude gli ID impostando la distanza a inf distances[:, exclude_ids] = float('inf') #escludi le ancore distances = distances[num_views:] k_nearest_indices = torch.topk(distances, k_near_views, largest=False).indices return k_nearest_indices def __len__(self): return len(self.data) def check_view_validaty(self, path_img): img = np.array(Image.open(os.path.join(path_img))) #check if transparent alpha = img[..., 3] is_fully_transparent = np.all(alpha == 0) return not(is_fully_transparent) def __getitem__(self, index): #{'obj_dir': path to folder uid, 'caption': ""} sample = self.data[index] obj_dir = sample["obj_dir"] with open(osp(obj_dir, "meta.json"), "r") as f: meta = json.load(f) # no use caption = sample["caption"] # camera intrinsics fov = meta["camera_angle_x"] focal_length = 0.5 * 1 / np.tan(0.5 * fov) # FIXME: hard-code intrinsics = np.array( [ [focal_length, 0, 0.5], [0, focal_length, 0.5], [0, 0, 1], ], dtype=np.float32, ) intrinsics = torch.from_numpy(intrinsics) intrinsics_4x4 = torch.zeros(4, 4) intrinsics_4x4[:3, :3] = intrinsics intrinsics_4x4[3, 3] = 1.0 # sample or select ref view ids num_views_all = len(meta["locations"]) if self.ref_view_mode == "random": check = False c = 0 while not(check): ref_view_id = np.random.randint(num_views_all) #chech if ref view is not transparent check = self.check_view_validaty(os.path.join(obj_dir, meta["locations"][ref_view_id]["frames"][0]["name"])) c=c+1 if c == self.num_views: #DA ELIMINARE print(f"The file at {obj_dir} directory do not contain any valid image....") elif isinstance(self.ref_view_mode, int): ref_view_id = self.ref_view_mode else: raise NotImplementedError # sample or select anchor view ids num_views_all = len(meta["locations"]) if self.sample_views_mode == "random": view_ids = np.random.choice( num_views_all, self.num_views - 1, replace=False ) elif isinstance(self.sample_views_mode, list) or isinstance( self.sample_views_mode, ListConfig ): assert len(self.sample_views_mode) == self.num_views - 1, "Invalid view_ids" view_ids = self.sample_views_mode else: raise NotImplementedError # sample or select derived view ids if self.sample_derived_views_mode == "random": derived_view_ids = np.random.choice( num_views_all, self.num_derived_views, replace=False ) elif isinstance(self.sample_derived_views_mode, list) or isinstance( self.sample_derived_views_mode, ListConfig ): derived_view_ids = self.sample_derived_views_mode else: raise NotImplementedError #insert reference view in the first position view_ids = np.insert(view_ids, 0, ref_view_id) #insert derived views in the last positions view_ids = np.insert(view_ids, self.num_views, derived_view_ids) locations = [meta["locations"][i] for i in view_ids] # load images, elevations, azimuths, c2w_matrixs bg_color = self.get_bg_color(self.bg_color) img_paths, img_tensors, elevations, azimuths, c2w_matrixs = [], [], [], [], [] for loc in locations: img_path = osp(obj_dir, loc["frames"][0]["name"]) img = self.load_image(img_path, bg_color, return_type="pt").permute(2, 0, 1) img_tensors.append(img) img_paths.append(img_path) elevations.append(loc["elevation"]) azimuths.append(loc["azimuth"]) c2w_matrixs.append(loc["transform_matrix"]) # concat and stack img_tensors = torch.stack(img_tensors, dim=0).float() # (Nv, 3, H, W) elevations = torch.tensor(elevations).float() # (Nv,) azimuths = torch.tensor(azimuths).float() # (Nv,) c2w_matrixs = torch.tensor(c2w_matrixs).float() # (Nv, 4, 4) # blender to opencv c2w_matrixs[:, :, 1:3] *= -1 intrinsics_matrixs = intrinsics.unsqueeze(0).repeat( self.num_views + self.num_derived_views, 1, 1 ) # (Nv, 3, 3) intrinsics_matrixs_4x4 = intrinsics_4x4.unsqueeze(0).repeat( self.num_views + self.num_derived_views, 1, 1 ) # (Nv, 4, 4) # flatten intrinsics_matrixs_4x4 and c2w_matrixs to (Nv, 16), and concat them intrinsics_matrixs_4x4 = intrinsics_matrixs_4x4.reshape(self.num_views + self.num_derived_views, 16) c2w_matrixs = c2w_matrixs.reshape(self.num_views + self.num_derived_views, 16) camera_params = torch.cat((intrinsics_matrixs_4x4, c2w_matrixs), dim=1) c2w_matrixs = c2w_matrixs.reshape(self.num_views + self.num_derived_views, 4, 4) # find nearest views if self.k_near_views > self.num_views: raise ValueError("k_near_views should be no larger than num_views") else: k_near_indices = self.get_k_near_anchors( elevations[1:], azimuths[1:], self.k_near_views, self.num_views - 1, exclude_ids=np.arange(self.num_views-1, self.num_views-1 + self.num_derived_views) ) # NOTE: we use the 1st view as the reference view #k_near_indices = -1 d_elevations = (elevations - elevations[0:1]).reshape(-1, 1) d_azimuths = (azimuths - azimuths[0:1]).reshape(-1, 1) % (2 * math.pi) if self.use_abs_elevation: # stable zero123 use abs_elevation as distance input distances = elevations[0:1].repeat(self.num_views + self.num_derived_views).reshape(-1, 1) else: distances = torch.zeros_like(d_elevations) return { "view_ids": torch.tensor(view_ids), "images": img_tensors[1:], "image_0": img_tensors[0], "elevations": d_elevations[1:], "azimuths": d_azimuths[1:], "distances": distances[1:], "c2w_matrixs": c2w_matrixs[1:], "intrinsics_matrixs": intrinsics_matrixs[1:], "cameras": camera_params[1:], "caption": caption, "k_near_indices": k_near_indices, "obj_dir": obj_dir.split('/')[-1], "num_anchor_images": self.num_views-1, "num_derived_images": self.num_derived_views } class MultiViewDataModule(LightningDataModule): def __init__( self, train_dataset: Dataset[Any], val_dataset: Dataset[Any], test_dataset: Dataset[Any], train_batch_size: int = 1, val_batch_size: int = 1, test_batch_size: int = 1, num_workers: Optional[int] = None, pin_memory: bool = True, ): super().__init__() # this line allows to access init params with 'self.hparams' attribute # also ensures init params will be stored in ckpt self.save_hyperparameters(logger=False) self.data_train = train_dataset self.data_val = val_dataset self.data_test = test_dataset self.num_workers = num_workers if num_workers else train_batch_size * 2 def prepare_data(self) -> None: # TODO: check if data is available pass def _dataloader( self, dataset: Dataset, batch_size: int, shuffle: bool ) -> DataLoader[Any]: return DataLoader( dataset, batch_size=batch_size, shuffle=shuffle, num_workers=self.num_workers, pin_memory=self.hparams.pin_memory, ) def train_dataloader(self) -> DataLoader[Any]: return DataLoader( self.data_train, batch_size=self.hparams.train_batch_size, shuffle=True, num_workers=self.num_workers, pin_memory=self.hparams.pin_memory, ) def val_dataloader(self) -> DataLoader[Any]: # support multiple validation datasets if isinstance(self.data_val, ListConfig): return [ self._dataloader(dataset, self.hparams.val_batch_size, False) for dataset in self.data_val ] elif isinstance(self.data_val, DictConfig): return [ self._dataloader(dataset, self.hparams.val_batch_size, False) for _, dataset in self.data_val.items() ] else: return self._dataloader(self.data_val, self.hparams.val_batch_size, False) def test_dataloader(self) -> DataLoader[Any]: # support multiple test datasets if isinstance(self.data_test, ListConfig): return [ self._dataloader(dataset, self.hparams.test_batch_size, False) for dataset in self.data_test ] elif isinstance(self.data_test, DictConfig): return [ self._dataloader(dataset, self.hparams.test_batch_size, False) for _, dataset in self.data_test.items() ] else: return self._dataloader(self.data_test, self.hparams.test_batch_size, False) if __name__ == "__main__": from torchvision.utils import save_image dataset = MultiViewDataset( root_dir="/mnt/pfs/data/render_lvis_hzh", num_views=17, k_near_views=4, bg_color="white", img_wh=(256, 256), ref_view_mode=2, sample_views_mode="random", caption_path="data/render_lvis_hzh/caption_train.txt", use_abs_elevation=True, ) sample = dataset[0] print(sample["view_ids"]) save_image((sample["image_0"] + 1.0) / 2, "temp_0.jpg", nrow=1, value_range=(0, 1)) save_image((sample["images"] + 1.0) / 2, "temp.jpg", nrow=8, value_range=(0, 1))