Spaces:
Sleeping
Sleeping
Download loomnet/data/multiview_infinity.py from FedericoGiulio/LoomNet: direct link, hf CLI and curl.
- Browser
- Download file 15.4 kB
-
https://huggingface.co/spaces/FedericoGiulio/LoomNet/resolve/main/loomnet/data/multiview_infinity.py
- Command line
-
hf download hf://spaces/FedericoGiulio/LoomNet/loomnet/data/multiview_infinity.py
-
curl -L -o multiview_infinity.py https://huggingface.co/spaces/FedericoGiulio/LoomNet/resolve/main/loomnet/data/multiview_infinity.py
15.4 kB
| import json | |
| import math | |
| import os | |
| from os.path import join as osp | |
| from typing import Any, Dict, List, Optional, Tuple, Union | |
| import random | |
| import numpy as np | |
| import torch | |
| from lightning import LightningDataModule | |
| from omegaconf import DictConfig, ListConfig | |
| from PIL import Image | |
| from torch.utils.data import DataLoader, Dataset | |
| class MultiViewDataset(Dataset): | |
| def __init__( | |
| self, | |
| type_of_dataset: str, | |
| root_dir: str, | |
| num_views: int, | |
| num_derived_views: int, | |
| bg_color: str, | |
| img_wh: Tuple[int, int], | |
| caption_path: str, | |
| ref_view_mode: Union[str, List] = "random", | |
| sample_views_mode: Union[str, List] = "random", | |
| sample_derived_views_mode: Union[str, List] = "random", | |
| k_near_views: Optional[int] = None, | |
| max_samples_per_category: Optional[int] = None, | |
| use_abs_elevation: bool = False, | |
| seed = 0 | |
| ): | |
| #data = [] | |
| # # Loop per ogni cartella di categoria | |
| # for category in os.listdir(root_dir): | |
| # category_path = os.path.join(root_dir, category) | |
| # if os.path.isdir(category_path): | |
| # uids = os.listdir(category_path) | |
| # #se validazione, prendiamo per semplicità solo due campioni | |
| # if max_samples_per_category is not None: | |
| # random.seed(seed) | |
| # uids = random.sample(uids, min(max_samples_per_category, len(uids))) | |
| # for uid in uids: | |
| # uid_path = os.path.join(root_dir, category, uid) | |
| # obj_data = {"obj_dir": uid_path, "caption": ""} | |
| # data.append(obj_data) | |
| data = [] | |
| # Loop per ogni cartella di categoria | |
| for category in os.listdir(root_dir): | |
| category_path = os.path.join(root_dir, category) | |
| if os.path.isdir(category_path): | |
| uids = [f for f in os.listdir(category_path) if not os.path.isfile(os.path.join(category_path, f))] #os.listdir(category_path) | |
| if len(uids)!=0: | |
| for uid in uids: | |
| uid_path = os.path.join(root_dir, category, uid) | |
| obj_data = {"obj_dir": uid_path, "caption": ""} | |
| data.append(obj_data) | |
| else: | |
| uid_path = os.path.join(root_dir, category) | |
| obj_data = {"obj_dir": uid_path, "caption": ""} | |
| data.append(obj_data) | |
| #da eliminare (per velocità prendo solo i primi 8) | |
| if type_of_dataset=="validation": | |
| data = data[:144] | |
| self.data = data | |
| self.num_views = num_views | |
| self.num_derived_views = num_derived_views | |
| self.bg_color = bg_color | |
| self.img_wh = img_wh | |
| self.ref_view_mode = ref_view_mode | |
| self.sample_views_mode = sample_views_mode | |
| self.sample_derived_views_mode = sample_derived_views_mode | |
| self.k_near_views = k_near_views if k_near_views else num_views | |
| self.use_abs_elevation = use_abs_elevation | |
| def get_bg_color(self, bg_color): | |
| if bg_color == "white": | |
| bg_color = np.array([1.0, 1.0, 1.0], dtype=np.float32) | |
| elif bg_color == "black": | |
| bg_color = np.array([0.0, 0.0, 0.0], dtype=np.float32) | |
| elif bg_color == "gray": | |
| bg_color = np.array([0.5, 0.5, 0.5], dtype=np.float32) | |
| elif bg_color == "random": | |
| bg_color = np.random.rand(3) | |
| elif bg_color == "random_gray": | |
| bg_color = np.random.rand(1) * 0.5 + 0.3 | |
| bg_color = np.array([bg_color] * 3, dtype=np.float32) | |
| elif isinstance(bg_color, float): | |
| bg_color = np.array([bg_color] * 3, dtype=np.float32) | |
| else: | |
| raise NotImplementedError | |
| return bg_color | |
| def load_image(self, img_path, bg_color, rescale=True, return_type="np"): | |
| # not using cv2 as may load in uint16 format | |
| # img = cv2.imread(img_path, cv2.IMREAD_UNCHANGED) # [0, 255] | |
| # img = cv2.resize(img, self.img_wh, interpolation=cv2.INTER_CUBIC) | |
| # pil always returns uint8 | |
| img = np.array(Image.open(img_path).resize(self.img_wh)) | |
| img = img.astype(np.float32) / 255.0 # [0, 1] | |
| if img.shape[-1] == 4: | |
| alpha = img[..., 3:4] | |
| img = img[..., :3] * alpha + bg_color * (1 - alpha) | |
| if rescale: | |
| img = img * 2.0 - 1.0 # to -1 ~ 1 | |
| if return_type == "np": | |
| pass | |
| elif return_type == "pt": | |
| img = torch.from_numpy(img) | |
| else: | |
| raise NotImplementedError | |
| return img | |
| def get_k_near_anchors(self, elevations, azimuths, k_near_views, num_views, exclude_ids=None): | |
| views = torch.cat((elevations.unsqueeze(1), azimuths.unsqueeze(1)), dim=1) | |
| distances = torch.cdist(views, views) | |
| # Imposta la diagonale a inf per non contare sé stessi | |
| distances.fill_diagonal_(float('inf')) | |
| if exclude_ids is not None: | |
| # Esclude gli ID impostando la distanza a inf | |
| distances[:, exclude_ids] = float('inf') | |
| #escludi le ancore | |
| distances = distances[num_views:] | |
| k_nearest_indices = torch.topk(distances, k_near_views, largest=False).indices | |
| return k_nearest_indices | |
| def __len__(self): | |
| return len(self.data) | |
| def check_view_validaty(self, path_img): | |
| img = np.array(Image.open(os.path.join(path_img))) | |
| #check if transparent | |
| alpha = img[..., 3] | |
| is_fully_transparent = np.all(alpha == 0) | |
| return not(is_fully_transparent) | |
| def __getitem__(self, index): | |
| #{'obj_dir': path to folder uid, 'caption': ""} | |
| sample = self.data[index] | |
| obj_dir = sample["obj_dir"] | |
| with open(osp(obj_dir, "meta.json"), "r") as f: | |
| meta = json.load(f) | |
| # no use | |
| caption = sample["caption"] | |
| # camera intrinsics | |
| fov = meta["camera_angle_x"] | |
| focal_length = 0.5 * 1 / np.tan(0.5 * fov) # FIXME: hard-code | |
| intrinsics = np.array( | |
| [ | |
| [focal_length, 0, 0.5], | |
| [0, focal_length, 0.5], | |
| [0, 0, 1], | |
| ], | |
| dtype=np.float32, | |
| ) | |
| intrinsics = torch.from_numpy(intrinsics) | |
| intrinsics_4x4 = torch.zeros(4, 4) | |
| intrinsics_4x4[:3, :3] = intrinsics | |
| intrinsics_4x4[3, 3] = 1.0 | |
| # sample or select ref view ids | |
| num_views_all = len(meta["locations"]) | |
| if self.ref_view_mode == "random": | |
| check = False | |
| c = 0 | |
| while not(check): | |
| ref_view_id = np.random.randint(num_views_all) | |
| #chech if ref view is not transparent | |
| check = self.check_view_validaty(os.path.join(obj_dir, meta["locations"][ref_view_id]["frames"][0]["name"])) | |
| c=c+1 | |
| if c == self.num_views: #DA ELIMINARE | |
| print(f"The file at {obj_dir} directory do not contain any valid image....") | |
| elif isinstance(self.ref_view_mode, int): | |
| ref_view_id = self.ref_view_mode | |
| else: | |
| raise NotImplementedError | |
| # sample or select anchor view ids | |
| num_views_all = len(meta["locations"]) | |
| if self.sample_views_mode == "random": | |
| view_ids = np.random.choice( | |
| num_views_all, self.num_views - 1, replace=False | |
| ) | |
| elif isinstance(self.sample_views_mode, list) or isinstance( | |
| self.sample_views_mode, ListConfig | |
| ): | |
| assert len(self.sample_views_mode) == self.num_views - 1, "Invalid view_ids" | |
| view_ids = self.sample_views_mode | |
| else: | |
| raise NotImplementedError | |
| # sample or select derived view ids | |
| if self.sample_derived_views_mode == "random": | |
| derived_view_ids = np.random.choice( | |
| num_views_all, self.num_derived_views, replace=False | |
| ) | |
| elif isinstance(self.sample_derived_views_mode, list) or isinstance( | |
| self.sample_derived_views_mode, ListConfig | |
| ): | |
| derived_view_ids = self.sample_derived_views_mode | |
| else: | |
| raise NotImplementedError | |
| #insert reference view in the first position | |
| view_ids = np.insert(view_ids, 0, ref_view_id) | |
| #insert derived views in the last positions | |
| view_ids = np.insert(view_ids, self.num_views, derived_view_ids) | |
| locations = [meta["locations"][i] for i in view_ids] | |
| # load images, elevations, azimuths, c2w_matrixs | |
| bg_color = self.get_bg_color(self.bg_color) | |
| img_paths, img_tensors, elevations, azimuths, c2w_matrixs = [], [], [], [], [] | |
| for loc in locations: | |
| img_path = osp(obj_dir, loc["frames"][0]["name"]) | |
| img = self.load_image(img_path, bg_color, return_type="pt").permute(2, 0, 1) | |
| img_tensors.append(img) | |
| img_paths.append(img_path) | |
| elevations.append(loc["elevation"]) | |
| azimuths.append(loc["azimuth"]) | |
| c2w_matrixs.append(loc["transform_matrix"]) | |
| # concat and stack | |
| img_tensors = torch.stack(img_tensors, dim=0).float() # (Nv, 3, H, W) | |
| elevations = torch.tensor(elevations).float() # (Nv,) | |
| azimuths = torch.tensor(azimuths).float() # (Nv,) | |
| c2w_matrixs = torch.tensor(c2w_matrixs).float() # (Nv, 4, 4) | |
| # blender to opencv | |
| c2w_matrixs[:, :, 1:3] *= -1 | |
| intrinsics_matrixs = intrinsics.unsqueeze(0).repeat( | |
| self.num_views + self.num_derived_views, 1, 1 | |
| ) # (Nv, 3, 3) | |
| intrinsics_matrixs_4x4 = intrinsics_4x4.unsqueeze(0).repeat( | |
| self.num_views + self.num_derived_views, 1, 1 | |
| ) # (Nv, 4, 4) | |
| # flatten intrinsics_matrixs_4x4 and c2w_matrixs to (Nv, 16), and concat them | |
| intrinsics_matrixs_4x4 = intrinsics_matrixs_4x4.reshape(self.num_views + self.num_derived_views, 16) | |
| c2w_matrixs = c2w_matrixs.reshape(self.num_views + self.num_derived_views, 16) | |
| camera_params = torch.cat((intrinsics_matrixs_4x4, c2w_matrixs), dim=1) | |
| c2w_matrixs = c2w_matrixs.reshape(self.num_views + self.num_derived_views, 4, 4) | |
| # find nearest views | |
| if self.k_near_views > self.num_views: | |
| raise ValueError("k_near_views should be no larger than num_views") | |
| else: | |
| k_near_indices = self.get_k_near_anchors( | |
| elevations[1:], azimuths[1:], self.k_near_views, self.num_views - 1, exclude_ids=np.arange(self.num_views-1, self.num_views-1 + self.num_derived_views) | |
| ) # NOTE: we use the 1st view as the reference view | |
| #k_near_indices = -1 | |
| d_elevations = (elevations - elevations[0:1]).reshape(-1, 1) | |
| d_azimuths = (azimuths - azimuths[0:1]).reshape(-1, 1) % (2 * math.pi) | |
| if self.use_abs_elevation: # stable zero123 use abs_elevation as distance input | |
| distances = elevations[0:1].repeat(self.num_views + self.num_derived_views).reshape(-1, 1) | |
| else: | |
| distances = torch.zeros_like(d_elevations) | |
| return { | |
| "view_ids": torch.tensor(view_ids), | |
| "images": img_tensors[1:], | |
| "image_0": img_tensors[0], | |
| "elevations": d_elevations[1:], | |
| "azimuths": d_azimuths[1:], | |
| "distances": distances[1:], | |
| "c2w_matrixs": c2w_matrixs[1:], | |
| "intrinsics_matrixs": intrinsics_matrixs[1:], | |
| "cameras": camera_params[1:], | |
| "caption": caption, | |
| "k_near_indices": k_near_indices, | |
| "obj_dir": obj_dir.split('/')[-1], | |
| "num_anchor_images": self.num_views-1, | |
| "num_derived_images": self.num_derived_views | |
| } | |
| class MultiViewDataModule(LightningDataModule): | |
| def __init__( | |
| self, | |
| train_dataset: Dataset[Any], | |
| val_dataset: Dataset[Any], | |
| test_dataset: Dataset[Any], | |
| train_batch_size: int = 1, | |
| val_batch_size: int = 1, | |
| test_batch_size: int = 1, | |
| num_workers: Optional[int] = None, | |
| pin_memory: bool = True, | |
| ): | |
| super().__init__() | |
| # this line allows to access init params with 'self.hparams' attribute | |
| # also ensures init params will be stored in ckpt | |
| self.save_hyperparameters(logger=False) | |
| self.data_train = train_dataset | |
| self.data_val = val_dataset | |
| self.data_test = test_dataset | |
| self.num_workers = num_workers if num_workers else train_batch_size * 2 | |
| def prepare_data(self) -> None: | |
| # TODO: check if data is available | |
| pass | |
| def _dataloader( | |
| self, dataset: Dataset, batch_size: int, shuffle: bool | |
| ) -> DataLoader[Any]: | |
| return DataLoader( | |
| dataset, | |
| batch_size=batch_size, | |
| shuffle=shuffle, | |
| num_workers=self.num_workers, | |
| pin_memory=self.hparams.pin_memory, | |
| ) | |
| def train_dataloader(self) -> DataLoader[Any]: | |
| return DataLoader( | |
| self.data_train, | |
| batch_size=self.hparams.train_batch_size, | |
| shuffle=True, | |
| num_workers=self.num_workers, | |
| pin_memory=self.hparams.pin_memory, | |
| ) | |
| def val_dataloader(self) -> DataLoader[Any]: | |
| # support multiple validation datasets | |
| if isinstance(self.data_val, ListConfig): | |
| return [ | |
| self._dataloader(dataset, self.hparams.val_batch_size, False) | |
| for dataset in self.data_val | |
| ] | |
| elif isinstance(self.data_val, DictConfig): | |
| return [ | |
| self._dataloader(dataset, self.hparams.val_batch_size, False) | |
| for _, dataset in self.data_val.items() | |
| ] | |
| else: | |
| return self._dataloader(self.data_val, self.hparams.val_batch_size, False) | |
| def test_dataloader(self) -> DataLoader[Any]: | |
| # support multiple test datasets | |
| if isinstance(self.data_test, ListConfig): | |
| return [ | |
| self._dataloader(dataset, self.hparams.test_batch_size, False) | |
| for dataset in self.data_test | |
| ] | |
| elif isinstance(self.data_test, DictConfig): | |
| return [ | |
| self._dataloader(dataset, self.hparams.test_batch_size, False) | |
| for _, dataset in self.data_test.items() | |
| ] | |
| else: | |
| return self._dataloader(self.data_test, self.hparams.test_batch_size, False) | |
| if __name__ == "__main__": | |
| from torchvision.utils import save_image | |
| dataset = MultiViewDataset( | |
| root_dir="/mnt/pfs/data/render_lvis_hzh", | |
| num_views=17, | |
| k_near_views=4, | |
| bg_color="white", | |
| img_wh=(256, 256), | |
| ref_view_mode=2, | |
| sample_views_mode="random", | |
| caption_path="data/render_lvis_hzh/caption_train.txt", | |
| use_abs_elevation=True, | |
| ) | |
| sample = dataset[0] | |
| print(sample["view_ids"]) | |
| save_image((sample["image_0"] + 1.0) / 2, "temp_0.jpg", nrow=1, value_range=(0, 1)) | |
| save_image((sample["images"] + 1.0) / 2, "temp.jpg", nrow=8, value_range=(0, 1)) | |