diff --git a/MaskClustering b/MaskClustering deleted file mode 120000 index 2f585f256ceaca2d39b42aa1c2da157874ef4b2e..0000000000000000000000000000000000000000 --- a/MaskClustering +++ /dev/null @@ -1 +0,0 @@ -../Indoor/MaskClustering/ \ No newline at end of file diff --git a/MaskClustering/arkit_gt_prep.py b/MaskClustering/arkit_gt_prep.py new file mode 100644 index 0000000000000000000000000000000000000000..d3ea1cc938dcb819031fa66901201ed1445b87bc --- /dev/null +++ b/MaskClustering/arkit_gt_prep.py @@ -0,0 +1,154 @@ +import pickle +from pathlib import Path +import torch +import cv2 +import numpy as np +import os +import open3d as o3d +from tqdm import tqdm + +def process_scene(scene_params, target_depth_shape=None): + images = scene_params["image_files"] + poses = scene_params["poses"] + depths = scene_params["depths"] + Ks = scene_params["Ks"] + pts3d = scene_params["pts3d"] + im_confs = scene_params["im_conf"] + print(scene_params.keys()) + im_shapes = scene_params["imshapes"] + im_shape = im_shapes[0] + + image_hw = cv2.imread(images[0]).shape[:2] + image_scale = np.ones((3, 3)) + image_scale[0] *= image_hw[1] / im_shape[1] + image_scale[1] *= image_hw[0] / im_shape[0] + + if target_depth_shape is None: + target_depth_shape = image_hw + + + depth_scale = np.ones((3, 3)) + depth_scale[0] *= target_depth_shape[1] / im_shape[1] + depth_scale[1] *= target_depth_shape[0] / im_shape[0] + + data = [ + { + "image_path": image, + "pose": pose, + "depth": cv2.resize(depth.numpy(), target_depth_shape[::-1], interpolation=cv2.INTER_LINEAR), + "source_K": K, + "image_K": K * image_scale, + "depth_K": K * depth_scale, + "pts3d": pts, + "im_conf": im_conf, + "im_shape_target": image_hw, + "depth_shape_target": target_depth_shape, + "shape_original": im_shape, + } for image, pose, depth, K, pts, im_conf in zip(images, poses, depths, Ks, pts3d, im_confs) + ] + + return data + + +def export_scene(scene_id, data, processing_args): + out_path = processing_args["out_dir"] / scene_id + K_color = data[0]["image_K"] + K_depth = data[0]["depth_K"] + + def proc_k(K): + res = np.eye(4) + res[:3, :3] = K[:3, :3] + return res + + K_color = proc_k(K_color) + K_depth = proc_k(K_depth) + intrinsics_path = out_path / "intrinsic" + intrinsics_path.mkdir(parents=True, exist_ok=True) + np.savetxt(intrinsics_path / "intrinsic_color.txt", K_color) + np.savetxt(intrinsics_path / "intrinsic_depth.txt", K_depth) + + np.savetxt(intrinsics_path / "extrinsic_color.txt", np.eye(4)) + np.savetxt(intrinsics_path / "extrinsic_depth.txt", np.eye(4)) + + for i, item in enumerate(data): + img_name = Path(item["image_path"]).stem + image_path = out_path / "color" / f"{img_name}.jpg" + image_path.parent.mkdir(parents=True, exist_ok=True) + try: + os.symlink(item["image_path"], image_path) + except FileExistsError: + pass + + + depth_path = out_path / "depth" / f"{img_name}.png" + depth_path.parent.mkdir(parents=True, exist_ok=True) + try: + os.symlink(item["depth_path"], depth_path) + except FileExistsError: + pass + + pose_path = out_path / "pose" / f"{img_name}.txt" + pose_path.parent.mkdir(parents=True, exist_ok=True) + np.savetxt(pose_path, item["pose"]) + + try: + os.symlink(item["pts3d_path"], out_path / f"{scene_id}_vh_clean_2.ply") + except FileExistsError: + pass + + + + + + +processing_args = { + "confidence_threshold": 1, + "voxel_size": 0.025, + "out_dir": Path("data/arkit_gt_train/processed"), +} + + +val_path = Path("../") / "OKNO/data/arkitscenes/arkitscenes_offline_infos_train.pkl" +out_dir = Path("data/arkit_gt/processed") +with open(val_path, "rb") as f: + data = pickle.load(f) + +data_list = data["data_list"] +val_scenes = [scene["lidar_points"]["lidar_path"] for scene in data_list][:2500] +def extract_name(item): + return item.split("_")[0] +val_scenes = [extract_name(scene) for scene in val_scenes] + +scenes_path = Path("/workspace-SR006.nfs2/datasets/arkitscenes/offline_prepared_data/posed_images") +pcd_path = Path("/workspace-SR006.nfs2/datasets/arkit_data/3dod/Training/") +scene = val_scenes[0] +num_images = 160 + + +for scene in tqdm(val_scenes): + try: + if (processing_args["out_dir"] / scene).exists(): + continue + scene_path = scenes_path / scene + + colors = sorted(scene_path.glob("*.jpg")) + if len(colors) > num_images: + indices = np.linspace(0, len(colors) - 1, num_images).astype(int) + colors = [colors[i] for i in indices] + depths = [a.parent / (a.stem + ".png") for a in colors] + poses = [a.parent / (a.stem + ".txt") for a in colors] + K = np.loadtxt(scene_path / "intrinsic.txt") + + scene_params = [{ + "image_path": image, + "pose": np.loadtxt(pose), + "depth_path": depth, + "source_K": K, + "image_K": K, + "depth_K": K, + "pts3d_path": pcd_path / scene / f"{scene}_3dod_mesh.ply", + } for image, depth, pose in zip(colors, depths, poses)] + export_scene(scene, scene_params, processing_args) + except Exception as e: + print(e) + continue \ No newline at end of file diff --git a/MaskClustering/arkit_prep.py b/MaskClustering/arkit_prep.py new file mode 100644 index 0000000000000000000000000000000000000000..e3e59750122a63d4884316ee8af7cfdad668309a --- /dev/null +++ b/MaskClustering/arkit_prep.py @@ -0,0 +1,137 @@ +import pickle +from pathlib import Path +import torch +import cv2 +import numpy as np +import os +import open3d as o3d +from tqdm import tqdm + +def process_scene(scene_params, target_depth_shape=None): + images = scene_params["image_files"] + poses = scene_params["poses"] + depths = scene_params["depths"] + Ks = scene_params["Ks"] + pts3d = scene_params["pts3d"] + im_confs = scene_params["im_conf"] + print(scene_params.keys()) + im_shapes = scene_params["imshapes"] + im_shape = im_shapes[0] + + image_hw = cv2.imread(images[0]).shape[:2] + image_scale = np.ones((3, 3)) + image_scale[0] *= image_hw[1] / im_shape[1] + image_scale[1] *= image_hw[0] / im_shape[0] + + if target_depth_shape is None: + target_depth_shape = image_hw + + + depth_scale = np.ones((3, 3)) + depth_scale[0] *= target_depth_shape[1] / im_shape[1] + depth_scale[1] *= target_depth_shape[0] / im_shape[0] + + data = [ + { + "image_path": image, + "pose": pose, + "depth": cv2.resize(depth.numpy(), target_depth_shape[::-1], interpolation=cv2.INTER_LINEAR), + "source_K": K, + "image_K": K * image_scale, + "depth_K": K * depth_scale, + "pts3d": pts, + "im_conf": im_conf, + "im_shape_target": image_hw, + "depth_shape_target": target_depth_shape, + "shape_original": im_shape, + } for image, pose, depth, K, pts, im_conf in zip(images, poses, depths, Ks, pts3d, im_confs) + ] + + return data + + +def export_scene(scene_id, scene_params, processing_args): + data = process_scene(scene_params) + out_path = processing_args["out_dir"] / scene_id + K_color = data[0]["image_K"] + K_depth = data[0]["depth_K"] + + def proc_k(K): + res = np.eye(4) + res[:3, :3] = K[:3, :3] + return res + + K_color = proc_k(K_color) + K_depth = proc_k(K_depth) + intrinsics_path = out_path / "intrinsic" + intrinsics_path.mkdir(parents=True, exist_ok=True) + np.savetxt(intrinsics_path / "intrinsic_color.txt", K_color) + np.savetxt(intrinsics_path / "intrinsic_depth.txt", K_depth) + + np.savetxt(intrinsics_path / "extrinsic_color.txt", np.eye(4)) + np.savetxt(intrinsics_path / "extrinsic_depth.txt", np.eye(4)) + all_pts = [] + all_colors = [] + for i, item in enumerate(data): + img_name = Path(item["image_path"]).stem + image_path = out_path / "color" / f"{img_name}.jpg" + image_path.parent.mkdir(parents=True, exist_ok=True) + try: + os.symlink(item["image_path"], image_path) + except FileExistsError: + pass + image = cv2.imread(item["image_path"]) + image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB) + image = cv2.resize(image, item['shape_original'][::-1]) / 255. + + + depth_path = out_path / "depth" / f"{img_name}.png" + depth_path.parent.mkdir(parents=True, exist_ok=True) + cv2.imwrite(depth_path, (item["depth"] * 1000).astype(np.uint16)) + + pose_path = out_path / "pose" / f"{img_name}.txt" + pose_path.parent.mkdir(parents=True, exist_ok=True) + np.savetxt(pose_path, item["pose"]) + pts = item["pts3d"][item["im_conf"] > processing_args["confidence_threshold"]] + image = image[item["im_conf"] > processing_args["confidence_threshold"]] + + all_pts.append(pts.view(-1, 3)) + all_colors.append(image.reshape(-1, 3)) + all_pts = np.concatenate(all_pts, axis=0) + all_colors = np.concatenate(all_colors, axis=0) + + pcd = o3d.geometry.PointCloud() + pcd.points = o3d.utility.Vector3dVector(all_pts) + pcd.colors = o3d.utility.Vector3dVector(all_colors) + pcd = pcd.voxel_down_sample(voxel_size=processing_args["voxel_size"]) + o3d.io.write_point_cloud(out_path / f"{scene_id}_vh_clean_2.ply", pcd) + + + + + + +processing_args = { + "confidence_threshold": 1, + "voxel_size": 0.025, + "out_dir": Path("data/arkit_dust3r_posed/processed"), +} + + +val_path = Path("../") / "OKNO/data/arkitscenes/arkitscenes_offline_infos_val.pkl" +out_dir = Path("data/arkit_dust3r_posed/processed") +with open(val_path, "rb") as f: + data = pickle.load(f) + +data_list = data["data_list"] +val_scenes = [scene["lidar_points"]["lidar_path"] for scene in data_list] +def extract_name(item): + return item.split("_")[0] +val_scenes = [extract_name(scene) for scene in val_scenes] + +dut3r_path = Path("/home/jovyan/users/lemeshko/Indoor/DUSt3R/res/arkit_posed") + +for scene in tqdm(val_scenes): + scene_path = dut3r_path / scene + scene_params = torch.load(scene_path / "scene_params.pt") + export_scene(scene, scene_params, processing_args) \ No newline at end of file diff --git a/MaskClustering/arkit_vggt_prep.py b/MaskClustering/arkit_vggt_prep.py new file mode 100644 index 0000000000000000000000000000000000000000..9eb33456fa5c584d65796181661f5af0ae950c29 --- /dev/null +++ b/MaskClustering/arkit_vggt_prep.py @@ -0,0 +1,154 @@ +import pickle +from pathlib import Path +import torch +import cv2 +import numpy as np +import os +import open3d as o3d +from tqdm import tqdm +from tqdm.contrib.concurrent import process_map, thread_map +from rec_utils.datasets import ARKitDataset, VGGTDataset +from rec_utils.aligner import build_aligner_1p1d +from rec_utils.datasets.arkit.utils import rotate_image + +def process_scene(scene_params, target_depth_shape=None): + images = scene_params["image_files"] + poses = scene_params["poses"] + depths = scene_params["depths"] + Ks = scene_params["Ks"] + pts3d = scene_params["pts3d"] + im_confs = scene_params["im_conf"] + print(scene_params.keys()) + im_shapes = scene_params["imshapes"] + im_shape = im_shapes[0] + + image_hw = cv2.imread(images[0]).shape[:2] + image_scale = np.ones((3, 3)) + image_scale[0] *= image_hw[1] / im_shape[1] + image_scale[1] *= image_hw[0] / im_shape[0] + + if target_depth_shape is None: + target_depth_shape = image_hw + + + depth_scale = np.ones((3, 3)) + depth_scale[0] *= target_depth_shape[1] / im_shape[1] + depth_scale[1] *= target_depth_shape[0] / im_shape[0] + + data = [ + { + "image_path": image, + "pose": pose, + "depth": cv2.resize(depth.numpy(), target_depth_shape[::-1], interpolation=cv2.INTER_LINEAR), + "source_K": K, + "image_K": K * image_scale, + "depth_K": K * depth_scale, + "pts3d": pts, + "im_conf": im_conf, + "im_shape_target": image_hw, + "depth_shape_target": target_depth_shape, + "shape_original": im_shape, + } for image, pose, depth, K, pts, im_conf in zip(images, poses, depths, Ks, pts3d, im_confs) + ] + + return data + +def es_wrap(data): + try: + return export_scene(data) + except Exception as e: + print(e) + return None + +def export_scene(data): + vggt_dataset, arkit_dataset, i, out_dir, processing_args = data + vggt_scene = vggt_dataset[i] + scene_id = vggt_scene.id + arkit_scene = arkit_dataset[scene_id] + out_path = out_dir / scene_id + out_path.mkdir(parents=True, exist_ok=True) + # if os.path.exists(out_path / f'{vggt_scene.id}_vh_clean_2.ply'): + # return + arkit_scene.frames = arkit_scene.frames[-100:] + aligner = build_aligner_1p1d(source_scene=vggt_scene, target_scene=arkit_scene) + scene = aligner.align(vggt_scene, inplace=True) + # scene = arkit_scene + + K_color = arkit_scene[0].image_intrinsics + K_depth = arkit_scene[0].depth_intrinsics + intrinsics_path = out_path / "intrinsic" + intrinsics_path.mkdir(parents=True, exist_ok=True) + color_path = out_path / "color" + color_path.mkdir(parents=True, exist_ok=True) + depth_path = out_path / "depth" + depth_path.mkdir(parents=True, exist_ok=True) + pose_path = out_path / "pose" + pose_path.mkdir(parents=True, exist_ok=True) + np.savetxt(intrinsics_path / "intrinsic_color.txt", K_color) + np.savetxt(intrinsics_path / "intrinsic_depth.txt", K_depth) + + np.savetxt(intrinsics_path / "extrinsic_color.txt", np.eye(4)) + np.savetxt(intrinsics_path / "extrinsic_depth.txt", np.eye(4)) + tsdffusion = o3d.pipelines.integration.ScalableTSDFVolume( + voxel_length=0.025, + sdf_trunc=0.1, + color_type=o3d.pipelines.integration.TSDFVolumeColorType.RGB8 + ) + print("rotation angle", arkit_scene.rotation_angle) + for frame in tqdm(scene.frames): + # image = rotate_image(frame.image, arkit_scene.rotation_angle) + # # image = frame.image + # h, w = image.shape[:2] + # depth = frame.depth.astype(np.float32) + # depth = cv2.resize(depth, (w, h), interpolation=cv2.INTER_LINEAR) + # color = o3d.geometry.Image(image) + np.savetxt(str(pose_path / f'{frame.frame_id}.txt'), frame.pose) + # cv2.imwrite(str(color_path / f'{frame.frame_id}.jpg'), image[..., ::-1]) + # cv2.imwrite(str(depth_path / f'{frame.frame_id}.png'), (depth * 1000.).astype(np.uint16)) + + # fx, fy = K_color[0, 0], K_color[1, 1] + # cx, cy = K_color[0, 2], K_color[1, 2] + + # dh, dw = depth.shape + # depth_o3d = o3d.geometry.Image(depth) + + # rgbd = o3d.geometry.RGBDImage.create_from_color_and_depth( + # color, depth_o3d, depth_trunc=10., convert_rgb_to_intensity=False, depth_scale=1.0 + # ) + # camera_o3d = o3d.camera.PinholeCameraIntrinsic(w, h, fx, fy, cx, cy) + # tsdffusion.integrate( + # rgbd, camera_o3d, + # np.linalg.inv(frame.pose), + # ) + # pc = tsdffusion.extract_point_cloud() + # pc.voxel_down_sample(voxel_size=0.025) + # o3d.io.write_point_cloud(str(out_path / f'{scene.id}_vh_clean_2.ply'), pc) + + + + + + + +vggt_dataset = VGGTDataset("/home/jovyan/users/bulat/workspace/3drec/vggt/output/arkit_new/") +arkit_dataset = ARKitDataset("/workspace-SR006.nfs2/datasets/arkitscenes/offline_prepared_data/posed_images/") +processing_args = { + "voxel_size": 0.025, + "out_dir": Path("data/arkit_dust3r_posed/processed"), +} + + +val_path = Path("../") / "OKNO/data/arkitscenes/arkitscenes_offline_infos_val.pkl" +out_dir = Path("data/arkit_vggt/processed") +with open(val_path, "rb") as f: + data = pickle.load(f) + +data_list = data["data_list"] +val_scenes = [scene["lidar_points"]["lidar_path"] for scene in data_list] +def extract_name(item): + return item.split("_")[0] +val_scenes = [extract_name(scene) for scene in val_scenes] +data = [(vggt_dataset, arkit_dataset, i, out_dir, processing_args) for i in range(len(vggt_dataset))] +thread_map(es_wrap, data, chunksize=128) + # break + diff --git a/MaskClustering/bins_build_pkl.py b/MaskClustering/bins_build_pkl.py new file mode 100644 index 0000000000000000000000000000000000000000..279e6b765d547c9836c00d40a879f31ac627a2ad --- /dev/null +++ b/MaskClustering/bins_build_pkl.py @@ -0,0 +1,36 @@ +import os +import pickle +from tqdm.auto import tqdm +from copy import deepcopy + + + +if __name__ == "__main__": + pred_path = \ + "/home/jovyan/users/bulat/workspace/3drec/Indoor/OKNO/data/arkitscenes/points_vggt/" + out_pkl_path = \ + "arkit_vggt_val_subset.pkl" + gt_pkl_path = \ + "/home/jovyan/users/bulat/workspace/3drec/Indoor/OKNO/data/arkitscenes/arkitscenes_offline_infos_val.pkl" + + + with open(gt_pkl_path, 'rb') as file: + gt_data = pickle.load(file) + + new_data = {"metainfo": gt_data["metainfo"]} + data_list = [] + + picked_scenes = [scene for scene in os.listdir(pred_path)] + num = 0 + for scene in tqdm(gt_data['data_list'][:10]): + scene_name = scene['lidar_points']['lidar_path'] + if scene_name not in picked_scenes: + print(f"Scene {scene_name} not found in {pred_path}") + continue + num += 1 + tmp_scene = deepcopy(scene) + data_list.append(tmp_scene) + print(f"Number of scenes: {num}") + new_data['data_list'] = data_list + with open(out_pkl_path, 'wb') as f: + pickle.dump(new_data, f) diff --git a/MaskClustering/build_pkl.py b/MaskClustering/build_pkl.py new file mode 100644 index 0000000000000000000000000000000000000000..3df6063af7d67730317a0e315f6ecb1573e4325b --- /dev/null +++ b/MaskClustering/build_pkl.py @@ -0,0 +1,36 @@ +import os +import pickle +from tqdm.auto import tqdm +from copy import deepcopy + + + +if __name__ == "__main__": + pred_path = \ + "/home/jovyan/users/bulat/workspace/3drec/Indoor/MaskClustering/data/prediction/arkit_gt_train" + out_pkl_path = \ + "arkit_gt_train.pkl" + gt_pkl_path = \ + "/home/jovyan/users/bulat/workspace/3drec/Indoor/OKNO/data/arkitscenes/arkitscenes_offline_infos_train.pkl" + + + with open(gt_pkl_path, 'rb') as file: + gt_data = pickle.load(file) + + new_data = {"metainfo": gt_data["metainfo"]} + data_list = [] + + picked_scenes = [scene[:-4] for scene in os.listdir(pred_path)] + num = 0 + for scene in tqdm(gt_data['data_list']): + scene_name = scene['lidar_points']['lidar_path'].split("_")[0] + if scene_name not in picked_scenes: + print(f"Scene {scene_name} not found in {pred_path}") + continue + num += 1 + tmp_scene = deepcopy(scene) + data_list.append(tmp_scene) + print(f"Number of scenes: {num}") + new_data['data_list'] = data_list + with open(out_pkl_path, 'wb') as f: + pickle.dump(new_data, f) diff --git a/MaskClustering/cluster_masks.py b/MaskClustering/cluster_masks.py new file mode 100644 index 0000000000000000000000000000000000000000..d4f3c3211a7400401bc33aaf6ab3e419112bd58f --- /dev/null +++ b/MaskClustering/cluster_masks.py @@ -0,0 +1,83 @@ +import numpy as np +import os +import cv2 +from pathlib import Path +import trimesh as tm +from sklearn.neighbors import KDTree +from tqdm import tqdm +from tqdm.contrib.concurrent import thread_map +from sklearn.cluster import DBSCAN + + +def load_scan(pcd_path): + pcd_data = np.fromfile(pcd_path, dtype=np.float32).reshape(-1, 6)[:, :3] + return pcd_data + +def process_scene(data): + scene_id, exp_name = data + pred_path = Path(f"data/prediction/scannet/click_sam/{scene_id}.npz") + out_path = Path(f"data/prediction/scannet/{exp_name}/{scene_id}.npz") + base_path = Path(f"/home/jovyan/users/lemeshko/scripts/gsam_result/yolo/{scene_id}") + source_path = Path(f"/home/jovyan/users/kolodiazhnyi/data/scannet/posed_images/{scene_id}") + scan_path = Path(f"/home/jovyan/users/bulat/workspace/3drec/Indoor/OKNO/data/scannet200/points/{scene_id}.bin") + info_path = base_path / "infos.npy" + + # if out_path.exists(): + # return + vertices = load_scan(scan_path) + info_data = np.load(info_path, allow_pickle=True).item() + + base_data = np.load(pred_path, allow_pickle=True) + + total_points_masks = base_data['pred_masks'].T + + for i, mask in enumerate(total_points_masks): + mask = mask.astype(bool) + points = vertices[mask] + db = DBSCAN(eps=0.3, min_samples=10) + if len(points) == 0: + continue + labels = db.fit_predict(points) + + # labels = db.labels_ + n_clusters = len(set(labels)) - (1 if -1 in labels else 0) + if (labels == -1).all(): + continue + biggest_cluster_ind = np.argmax(np.unique(labels[labels != -1], return_counts=True)[1]) + res_mask = (labels == biggest_cluster_ind) & (labels != -1) + # print(f"{labels.shape} -> {res_mask.sum()}") + new_mask = np.zeros_like(mask) + new_mask[mask] = res_mask + total_points_masks[i] = new_mask + + new_data = { + k: v for k, v in base_data.items() + } + new_data['pred_masks'] = total_points_masks.T + + out_path.parent.mkdir(parents=True, exist_ok=True) + # vs = [] + # cs = [] + # for i in range(new_data['pred_masks'].shape[1]): + # os.makedirs(f"pred_masks", exist_ok=True) + # v = vertices[new_data['pred_masks'][:, i]] + # c = np.random.rand(3) + # c = np.repeat(c[np.newaxis, :], len(v), axis=0) + # vs.append(v) + # cs.append(c) + # tm.PointCloud(np.concatenate(vs, axis=0), colors=np.concatenate(cs, axis=0)).export(f"pred_masks/{scene_id}_mask.ply") + + print("uniques", np.unique(new_data['pred_masks'].sum(1)), [[k, v.shape] for k, v in new_data.items()]) + np.savez(out_path, **new_data) + + + +if __name__ == "__main__": + exp_name = "cluster_filtering_click_sam" + # scenes = np.loadtxt("/home/jovyan/users/bulat/workspace/3drec/Indoor/MaskClustering/splits/scannet.txt", dtype=str) + scenes = ["scene0011_00"] + data = [(scene, exp_name) for scene in scenes] + total_points_masks = thread_map(process_scene, data, chunksize=20) + + + diff --git a/MaskClustering/configs/arkit_dust3r_posed.json b/MaskClustering/configs/arkit_dust3r_posed.json new file mode 100644 index 0000000000000000000000000000000000000000..306066a64910fa0e2f2e19dda0ee2f1cf7432434 --- /dev/null +++ b/MaskClustering/configs/arkit_dust3r_posed.json @@ -0,0 +1,10 @@ +{ + "mask_visible_threshold": 0.3, + "undersegment_filter_threshold": 0.3, + "view_consensus_threshold": 0.9, + "contained_threshold": 0.8, + "point_filter_threshold": 0.5, + "dataset": "arkit_dust3r_posed", + "cropformer_path": "Mask2Former_hornet_3x_576d0b.pth", + "step": 10 +} \ No newline at end of file diff --git a/MaskClustering/configs/arkit_gt.json b/MaskClustering/configs/arkit_gt.json new file mode 100644 index 0000000000000000000000000000000000000000..84c0016ff290fe063da9a1ab48b0f9d0596d8923 --- /dev/null +++ b/MaskClustering/configs/arkit_gt.json @@ -0,0 +1,10 @@ +{ + "mask_visible_threshold": 0.3, + "undersegment_filter_threshold": 0.3, + "view_consensus_threshold": 0.9, + "contained_threshold": 0.8, + "point_filter_threshold": 0.5, + "dataset": "arkit_gt", + "cropformer_path": "Mask2Former_hornet_3x_576d0b.pth", + "step": 10 +} \ No newline at end of file diff --git a/MaskClustering/configs/arkit_gt_train.json b/MaskClustering/configs/arkit_gt_train.json new file mode 100644 index 0000000000000000000000000000000000000000..48035aff7ea21e4f27d562e4a2240b90851813d1 --- /dev/null +++ b/MaskClustering/configs/arkit_gt_train.json @@ -0,0 +1,10 @@ +{ + "mask_visible_threshold": 0.3, + "undersegment_filter_threshold": 0.3, + "view_consensus_threshold": 0.9, + "contained_threshold": 0.8, + "point_filter_threshold": 0.5, + "dataset": "arkit_gt_train", + "cropformer_path": "Mask2Former_hornet_3x_576d0b.pth", + "step": 10 +} \ No newline at end of file diff --git a/MaskClustering/configs/arkit_vggt.json b/MaskClustering/configs/arkit_vggt.json new file mode 100644 index 0000000000000000000000000000000000000000..7d29e1d9c0bf836f38e4b7a791a05b94659d3987 --- /dev/null +++ b/MaskClustering/configs/arkit_vggt.json @@ -0,0 +1,10 @@ +{ + "mask_visible_threshold": 0.3, + "undersegment_filter_threshold": 0.3, + "view_consensus_threshold": 0.9, + "contained_threshold": 0.8, + "point_filter_threshold": 0.5, + "dataset": "arkit_vggt", + "cropformer_path": "Mask2Former_hornet_3x_576d0b.pth", + "step": 10 +} \ No newline at end of file diff --git a/MaskClustering/configs/demo.json b/MaskClustering/configs/demo.json new file mode 100644 index 0000000000000000000000000000000000000000..c1203963d8aeb021cc335c742fb6de17af2325af --- /dev/null +++ b/MaskClustering/configs/demo.json @@ -0,0 +1,10 @@ +{ + "mask_visible_threshold": 0.3, + "undersegment_filter_threshold": 0.3, + "view_consensus_threshold": 0.9, + "contained_threshold": 0.8, + "point_filter_threshold": 0.5, + "dataset": "demo", + "cropformer_path": "/raid/miyan/ckpt/Mask2Former_hornet_3x_576d0b.pth", + "step": 10 +} \ No newline at end of file diff --git a/MaskClustering/configs/itw.json b/MaskClustering/configs/itw.json new file mode 100644 index 0000000000000000000000000000000000000000..b18e8edf65a354c469b7b371e44f205f82765c43 --- /dev/null +++ b/MaskClustering/configs/itw.json @@ -0,0 +1,10 @@ +{ + "mask_visible_threshold": 0.3, + "undersegment_filter_threshold": 0.3, + "view_consensus_threshold": 0.9, + "contained_threshold": 0.8, + "point_filter_threshold": 0.5, + "dataset": "itw", + "cropformer_path": "Mask2Former_hornet_3x_576d0b.pth", + "step": 10 +} \ No newline at end of file diff --git a/MaskClustering/configs/matterport3d.json b/MaskClustering/configs/matterport3d.json new file mode 100644 index 0000000000000000000000000000000000000000..871369778c43775ecd05aaa517e1e7c6c2c957d5 --- /dev/null +++ b/MaskClustering/configs/matterport3d.json @@ -0,0 +1,10 @@ +{ + "mask_visible_threshold": 0.3, + "undersegment_filter_threshold": 0.3, + "view_consensus_threshold": 0.9, + "contained_threshold": 0.8, + "point_filter_threshold": 0.5, + "dataset": "matterport3d", + "cropformer_path": "/raid/miyan/ckpt/Mask2Former_hornet_3x_576d0b.pth", + "step": 1 +} \ No newline at end of file diff --git a/MaskClustering/configs/scannet.json b/MaskClustering/configs/scannet.json new file mode 100644 index 0000000000000000000000000000000000000000..a16aa9503582783593bf90e6e73f89a61696150d --- /dev/null +++ b/MaskClustering/configs/scannet.json @@ -0,0 +1,10 @@ +{ + "mask_visible_threshold": 0.3, + "undersegment_filter_threshold": 0.3, + "view_consensus_threshold": 0.9, + "contained_threshold": 0.8, + "point_filter_threshold": 0.5, + "dataset": "scannet", + "cropformer_path": "Mask2Former_hornet_3x_576d0b.pth", + "step": 10 +} \ No newline at end of file diff --git a/MaskClustering/configs/scannet_dust3r_posed_15.json b/MaskClustering/configs/scannet_dust3r_posed_15.json new file mode 100644 index 0000000000000000000000000000000000000000..499353cc1194fd32ab17f00172eab4d2c187b34c --- /dev/null +++ b/MaskClustering/configs/scannet_dust3r_posed_15.json @@ -0,0 +1,10 @@ +{ + "mask_visible_threshold": 0.3, + "undersegment_filter_threshold": 0.3, + "view_consensus_threshold": 0.9, + "contained_threshold": 0.8, + "point_filter_threshold": 0.5, + "dataset": "scannet_dust3r_posed_15", + "cropformer_path": "Mask2Former_hornet_3x_576d0b.pth", + "step": 10 +} \ No newline at end of file diff --git a/MaskClustering/configs/scannet_dust3r_posed_25.json b/MaskClustering/configs/scannet_dust3r_posed_25.json new file mode 100644 index 0000000000000000000000000000000000000000..416351e7702d3ba06f61abd3ea32d06260410e2a --- /dev/null +++ b/MaskClustering/configs/scannet_dust3r_posed_25.json @@ -0,0 +1,10 @@ +{ + "mask_visible_threshold": 0.3, + "undersegment_filter_threshold": 0.3, + "view_consensus_threshold": 0.9, + "contained_threshold": 0.8, + "point_filter_threshold": 0.5, + "dataset": "scannet_dust3r_posed_25", + "cropformer_path": "Mask2Former_hornet_3x_576d0b.pth", + "step": 10 +} \ No newline at end of file diff --git a/MaskClustering/configs/scannet_dust3r_posed_35.json b/MaskClustering/configs/scannet_dust3r_posed_35.json new file mode 100644 index 0000000000000000000000000000000000000000..fe3e18fc26c1e77e23f02db867770054b2c519f7 --- /dev/null +++ b/MaskClustering/configs/scannet_dust3r_posed_35.json @@ -0,0 +1,10 @@ +{ + "mask_visible_threshold": 0.3, + "undersegment_filter_threshold": 0.3, + "view_consensus_threshold": 0.9, + "contained_threshold": 0.8, + "point_filter_threshold": 0.5, + "dataset": "scannet_dust3r_posed_35", + "cropformer_path": "Mask2Former_hornet_3x_576d0b.pth", + "step": 10 +} \ No newline at end of file diff --git a/MaskClustering/configs/scannet_dust3r_posed_35_bulat.json b/MaskClustering/configs/scannet_dust3r_posed_35_bulat.json new file mode 100644 index 0000000000000000000000000000000000000000..f183a1555fffde7b057964ea7965b039776a0969 --- /dev/null +++ b/MaskClustering/configs/scannet_dust3r_posed_35_bulat.json @@ -0,0 +1,10 @@ +{ + "mask_visible_threshold": 0.3, + "undersegment_filter_threshold": 0.3, + "view_consensus_threshold": 0.9, + "contained_threshold": 0.8, + "point_filter_threshold": 0.5, + "dataset": "scannet_dust3r_posed_35_bulat", + "cropformer_path": "Mask2Former_hornet_3x_576d0b.pth", + "step": 10 +} \ No newline at end of file diff --git a/MaskClustering/configs/scannet_dust3r_posed_45.json b/MaskClustering/configs/scannet_dust3r_posed_45.json new file mode 100644 index 0000000000000000000000000000000000000000..1852a1d416fdc5abe2a624b6e39f9fdcbe7d60ee --- /dev/null +++ b/MaskClustering/configs/scannet_dust3r_posed_45.json @@ -0,0 +1,10 @@ +{ + "mask_visible_threshold": 0.3, + "undersegment_filter_threshold": 0.3, + "view_consensus_threshold": 0.9, + "contained_threshold": 0.8, + "point_filter_threshold": 0.5, + "dataset": "scannet_dust3r_posed_45", + "cropformer_path": "Mask2Former_hornet_3x_576d0b.pth", + "step": 10 +} \ No newline at end of file diff --git a/MaskClustering/configs/scannet_dust3r_posed_45_andrey.json b/MaskClustering/configs/scannet_dust3r_posed_45_andrey.json new file mode 100644 index 0000000000000000000000000000000000000000..8d67f1544daab6ea812669e3cce1f5a8bb81e249 --- /dev/null +++ b/MaskClustering/configs/scannet_dust3r_posed_45_andrey.json @@ -0,0 +1,10 @@ +{ + "mask_visible_threshold": 0.3, + "undersegment_filter_threshold": 0.3, + "view_consensus_threshold": 0.9, + "contained_threshold": 0.8, + "point_filter_threshold": 0.5, + "dataset": "scannet_dust3r_posed_45_andrey", + "cropformer_path": "Mask2Former_hornet_3x_576d0b.pth", + "step": 10 +} \ No newline at end of file diff --git a/MaskClustering/configs/scannet_dust3r_posed_45_bulat.json b/MaskClustering/configs/scannet_dust3r_posed_45_bulat.json new file mode 100644 index 0000000000000000000000000000000000000000..1fc665dae4c456eafd3cd1a0813e678278218dc9 --- /dev/null +++ b/MaskClustering/configs/scannet_dust3r_posed_45_bulat.json @@ -0,0 +1,10 @@ +{ + "mask_visible_threshold": 0.3, + "undersegment_filter_threshold": 0.3, + "view_consensus_threshold": 0.9, + "contained_threshold": 0.8, + "point_filter_threshold": 0.5, + "dataset": "scannet_dust3r_posed_45_bulat", + "cropformer_path": "Mask2Former_hornet_3x_576d0b.pth", + "step": 10 +} \ No newline at end of file diff --git a/MaskClustering/configs/scannet_dust3r_unposed_15.json b/MaskClustering/configs/scannet_dust3r_unposed_15.json new file mode 100644 index 0000000000000000000000000000000000000000..c217c5371df8fb99ecd79f828cd66d4b260a5ffa --- /dev/null +++ b/MaskClustering/configs/scannet_dust3r_unposed_15.json @@ -0,0 +1,10 @@ +{ + "mask_visible_threshold": 0.3, + "undersegment_filter_threshold": 0.3, + "view_consensus_threshold": 0.9, + "contained_threshold": 0.8, + "point_filter_threshold": 0.5, + "dataset": "scannet_dust3r_unposed_15", + "cropformer_path": "Mask2Former_hornet_3x_576d0b.pth", + "step": 10 +} \ No newline at end of file diff --git a/MaskClustering/configs/scannet_dust3r_unposed_25.json b/MaskClustering/configs/scannet_dust3r_unposed_25.json new file mode 100644 index 0000000000000000000000000000000000000000..3b20bb9989003c1fc994558317bd4f3038334463 --- /dev/null +++ b/MaskClustering/configs/scannet_dust3r_unposed_25.json @@ -0,0 +1,10 @@ +{ + "mask_visible_threshold": 0.3, + "undersegment_filter_threshold": 0.3, + "view_consensus_threshold": 0.9, + "contained_threshold": 0.8, + "point_filter_threshold": 0.5, + "dataset": "scannet_dust3r_unposed_25", + "cropformer_path": "Mask2Former_hornet_3x_576d0b.pth", + "step": 10 +} \ No newline at end of file diff --git a/MaskClustering/configs/scannet_dust3r_unposed_35.json b/MaskClustering/configs/scannet_dust3r_unposed_35.json new file mode 100644 index 0000000000000000000000000000000000000000..e87d6197fad8135bc1a86685a2bc3c730e4f847f --- /dev/null +++ b/MaskClustering/configs/scannet_dust3r_unposed_35.json @@ -0,0 +1,10 @@ +{ + "mask_visible_threshold": 0.3, + "undersegment_filter_threshold": 0.3, + "view_consensus_threshold": 0.9, + "contained_threshold": 0.8, + "point_filter_threshold": 0.5, + "dataset": "scannet_dust3r_unposed_35", + "cropformer_path": "Mask2Former_hornet_3x_576d0b.pth", + "step": 10 +} \ No newline at end of file diff --git a/MaskClustering/configs/scannet_dust3r_unposed_45.json b/MaskClustering/configs/scannet_dust3r_unposed_45.json new file mode 100644 index 0000000000000000000000000000000000000000..15aa65a04f64c2d7817f79819fe4066dc735b375 --- /dev/null +++ b/MaskClustering/configs/scannet_dust3r_unposed_45.json @@ -0,0 +1,10 @@ +{ + "mask_visible_threshold": 0.3, + "undersegment_filter_threshold": 0.3, + "view_consensus_threshold": 0.9, + "contained_threshold": 0.8, + "point_filter_threshold": 0.5, + "dataset": "scannet_dust3r_unposed_45", + "cropformer_path": "Mask2Former_hornet_3x_576d0b.pth", + "step": 10 +} \ No newline at end of file diff --git a/MaskClustering/configs/scannetpp.json b/MaskClustering/configs/scannetpp.json new file mode 100644 index 0000000000000000000000000000000000000000..f5aa856f97b7e2e6e5f04fdf94938ac961a9ba9e --- /dev/null +++ b/MaskClustering/configs/scannetpp.json @@ -0,0 +1,10 @@ +{ + "mask_visible_threshold": 0.4, + "undersegment_filter_threshold": 0.2, + "view_consensus_threshold": 1, + "contained_threshold": 0.9, + "point_filter_threshold": 0.7, + "dataset": "scannetpp", + "cropformer_path": "/raid/miyan/ckpt/Mask2Former_hornet_3x_576d0b.pth", + "step": 2 +} \ No newline at end of file diff --git a/MaskClustering/configs/scannetpp_dust3r_filtered_depth.json b/MaskClustering/configs/scannetpp_dust3r_filtered_depth.json new file mode 100644 index 0000000000000000000000000000000000000000..fdf43b47e186ba2d46e5b2cd5de5c43661ce411a --- /dev/null +++ b/MaskClustering/configs/scannetpp_dust3r_filtered_depth.json @@ -0,0 +1,10 @@ +{ + "mask_visible_threshold": 0.4, + "undersegment_filter_threshold": 0.2, + "view_consensus_threshold": 1, + "contained_threshold": 0.9, + "point_filter_threshold": 0.7, + "dataset": "scannetpp_dust3r_filtered_depth", + "cropformer_path": "Mask2Former_hornet_3x_576d0b.pth", + "step": 2 +} \ No newline at end of file diff --git a/MaskClustering/configs/scannetpp_dust3r_posed.json b/MaskClustering/configs/scannetpp_dust3r_posed.json new file mode 100644 index 0000000000000000000000000000000000000000..43d996a6ca891874b2eb68c90ef618d8fe504ed5 --- /dev/null +++ b/MaskClustering/configs/scannetpp_dust3r_posed.json @@ -0,0 +1,10 @@ +{ + "mask_visible_threshold": 0.4, + "undersegment_filter_threshold": 0.2, + "view_consensus_threshold": 1, + "contained_threshold": 0.9, + "point_filter_threshold": 0.7, + "dataset": "scannetpp_dust3r_posed", + "cropformer_path": "Mask2Former_hornet_3x_576d0b.pth", + "step": 2 +} \ No newline at end of file diff --git a/MaskClustering/configs/scannetpp_dust3r_unposed.json b/MaskClustering/configs/scannetpp_dust3r_unposed.json new file mode 100644 index 0000000000000000000000000000000000000000..0c715338a917167736f5a064c9c84a75b2ac956f --- /dev/null +++ b/MaskClustering/configs/scannetpp_dust3r_unposed.json @@ -0,0 +1,10 @@ +{ + "mask_visible_threshold": 0.4, + "undersegment_filter_threshold": 0.2, + "view_consensus_threshold": 1, + "contained_threshold": 0.9, + "point_filter_threshold": 0.7, + "dataset": "scannetpp_dust3r_unposed", + "cropformer_path": "Mask2Former_hornet_3x_576d0b.pth", + "step": 2 +} \ No newline at end of file diff --git a/MaskClustering/configs/scannetpp_mapanything_posed.json b/MaskClustering/configs/scannetpp_mapanything_posed.json new file mode 100644 index 0000000000000000000000000000000000000000..cf332f694fc4b9f9ddf9fceddcd1963393854a3b --- /dev/null +++ b/MaskClustering/configs/scannetpp_mapanything_posed.json @@ -0,0 +1,10 @@ +{ + "mask_visible_threshold": 0.4, + "undersegment_filter_threshold": 0.2, + "view_consensus_threshold": 1, + "contained_threshold": 0.9, + "point_filter_threshold": 0.7, + "dataset": "scannetpp_mapanything_posed", + "cropformer_path": "Mask2Former_hornet_3x_576d0b.pth", + "step": 2 +} \ No newline at end of file diff --git a/MaskClustering/configs/scannetpp_v2_dust3r_posed.json b/MaskClustering/configs/scannetpp_v2_dust3r_posed.json new file mode 100644 index 0000000000000000000000000000000000000000..5180c65cd1a9ee94d520ce69d6da088bd5840930 --- /dev/null +++ b/MaskClustering/configs/scannetpp_v2_dust3r_posed.json @@ -0,0 +1,10 @@ +{ + "mask_visible_threshold": 0.4, + "undersegment_filter_threshold": 0.2, + "view_consensus_threshold": 1, + "contained_threshold": 0.9, + "point_filter_threshold": 0.7, + "dataset": "scannetpp_v2_dust3r_posed", + "cropformer_path": "Mask2Former_hornet_3x_576d0b.pth", + "step": 2 +} diff --git a/MaskClustering/configs/scannetpp_v2_dust3r_unposed.json b/MaskClustering/configs/scannetpp_v2_dust3r_unposed.json new file mode 100644 index 0000000000000000000000000000000000000000..1b93a2e9de43b052c1f0e29e4573153513f4177c --- /dev/null +++ b/MaskClustering/configs/scannetpp_v2_dust3r_unposed.json @@ -0,0 +1,10 @@ +{ + "mask_visible_threshold": 0.4, + "undersegment_filter_threshold": 0.2, + "view_consensus_threshold": 1, + "contained_threshold": 0.9, + "point_filter_threshold": 0.7, + "dataset": "scannetpp_v2_dust3r_unposed", + "cropformer_path": "Mask2Former_hornet_3x_576d0b.pth", + "step": 2 +} diff --git a/MaskClustering/configs/wild.json b/MaskClustering/configs/wild.json new file mode 100644 index 0000000000000000000000000000000000000000..d413435866e8b9c29f786d70a833902ac5c9b206 --- /dev/null +++ b/MaskClustering/configs/wild.json @@ -0,0 +1,10 @@ +{ + "mask_visible_threshold": 0.3, + "undersegment_filter_threshold": 0.3, + "view_consensus_threshold": 0.9, + "contained_threshold": 0.8, + "point_filter_threshold": 0.5, + "dataset": "wild", + "cropformer_path": "Mask2Former_hornet_3x_576d0b.pth", + "step": 10 +} \ No newline at end of file diff --git a/MaskClustering/dataset/demo.py b/MaskClustering/dataset/demo.py new file mode 100644 index 0000000000000000000000000000000000000000..28bd8cebac3b4c5c64e7ebfc3b05e31fb9f3cbdf --- /dev/null +++ b/MaskClustering/dataset/demo.py @@ -0,0 +1,100 @@ +import open3d as o3d +import numpy as np +import os +import cv2 +from evaluation.constants import SCANNET_LABELS, SCANNET_IDS + +class DemoDataset: + + def __init__(self, seq_name) -> None: + self.seq_name = seq_name + self.root = f'./data/demo/{seq_name}' + self.rgb_dir = f'{self.root}/color_640' + self.depth_dir = f'{self.root}/depth' + self.segmentation_dir = f'{self.root}/output/mask' + self.object_dict_dir = f'{self.root}/output/object' + self.point_cloud_path = f'{self.root}/{seq_name}_vh_clean_2.ply' + self.mesh_path = self.point_cloud_path + self.extrinsics_dir = f'{self.root}/pose' + + self.depth_scale = 1000.0 + self.image_size = (640, 480) + + + def get_frame_list(self, stride): + image_list = os.listdir(self.rgb_dir) + image_list = sorted(image_list, key=lambda x: int(x.split('.')[0])) + + end = int(image_list[-1].split('.')[0]) + 1 + frame_id_list = np.arange(0, end, stride) + return list(frame_id_list) + + + def get_intrinsics(self, frame_id): + intrinsic_path = f'{self.root}/intrinsic_640.txt' + intrinsics = np.loadtxt(intrinsic_path) + + intrinisc_cam_parameters = o3d.camera.PinholeCameraIntrinsic() + intrinisc_cam_parameters.set_intrinsics(640, 480, intrinsics[0, 0], intrinsics[1, 1], intrinsics[0, 2], intrinsics[1, 2]) + return intrinisc_cam_parameters + + + def get_extrinsic(self, frame_id): + pose_path = os.path.join(self.extrinsics_dir, str(frame_id) + '.txt') + pose = np.loadtxt(pose_path) + return pose + + + def get_depth(self, frame_id): + depth_path = os.path.join(self.depth_dir, str(frame_id) + '.png') + depth = cv2.imread(depth_path, -1) + depth = depth / self.depth_scale + depth = depth.astype(np.float32) + return depth + + + def get_rgb(self, frame_id, change_color=True): + rgb_path = os.path.join(self.rgb_dir, str(frame_id) + '.jpg') + rgb = cv2.imread(rgb_path) + + if change_color: + rgb = cv2.cvtColor(rgb, cv2.COLOR_BGR2RGB) + return rgb + + + def get_segmentation(self, frame_id, align_with_depth=False): + segmentation_path = os.path.join(self.segmentation_dir, f'{frame_id}.png') + if not os.path.exists(segmentation_path): + assert False, f"Segmentation not found: {segmentation_path}" + segmentation = cv2.imread(segmentation_path, cv2.IMREAD_UNCHANGED) + return segmentation + + + def get_frame_path(self, frame_id): + rgb_path = os.path.join(self.rgb_dir, str(frame_id) + '.jpg') + segmentation_path = os.path.join(self.segmentation_dir, f'{frame_id}.png') + return rgb_path, segmentation_path + + + def get_label_features(self): + label_features_dict = np.load(f'data/text_features/scannet.npy', allow_pickle=True).item() + return label_features_dict + + + def get_scene_points(self): + mesh = o3d.io.read_point_cloud(self.point_cloud_path) + vertices = np.asarray(mesh.points) + return vertices + + + def get_label_id(self): + self.class_id = SCANNET_IDS + self.class_label = SCANNET_LABELS + + self.label2id = {} + self.id2label = {} + for label, id in zip(self.class_label, self.class_id): + self.label2id[label] = id + self.id2label[id] = label + + return self.label2id, self.id2label \ No newline at end of file diff --git a/MaskClustering/dataset/matterport.py b/MaskClustering/dataset/matterport.py new file mode 100644 index 0000000000000000000000000000000000000000..3ba37a64b398e1b7702f006171ee665f5e7e16ec --- /dev/null +++ b/MaskClustering/dataset/matterport.py @@ -0,0 +1,137 @@ +import open3d as o3d +import numpy as np +import os +import cv2 +from evaluation.constants import MATTERPORT_LABELS, MATTERPORT_IDS + +class MatterportDataset: + def __init__(self, seq_name) -> None: + self.seq_name = seq_name + self.root = f'./data/matterport3d/scans/{seq_name}/{seq_name}' + self.rgb_dir = f'{self.root}/undistorted_color_images' + self.depth_dir = f'{self.root}/undistorted_depth_images' + self.cam_param_dir = f'{self.root}/undistorted_camera_parameters/{seq_name}.conf' + self.point_cloud_path = f'{self.root}/house_segmentations/{seq_name}.ply' + self.mesh_path = self.point_cloud_path + self.rgb_names, self.depth_names, self.intrinsics, self.extrinsics = \ + self._obtain_intr_extr() + + # output + self.segmentation_dir = f'{self.root}/output/mask/' + self.object_dict_dir = f'{self.root}/output/object' + + self.depth_scale = 4000.0 # (0.25mm per unit) 1u = 1/4000 m + self.image_size = (1280, 1024) + + + def get_frame_list(self, step): + image_list = [os.path.join(self.rgb_dir, rgb_name) for rgb_name in self.rgb_names] + + end = len(image_list) + frame_id_list = np.arange(0, end, step) + return list(frame_id_list) + + + def _obtain_intr_extr(self): + '''Obtain the intrinsic and extrinsic parameters of Matterport3D.''' + + with open(self.cam_param_dir, 'r') as file: + lines = file.readlines() + + def remove_items(test_list, item): + return [i for i in test_list if i != item] + + intrinsics = [] + extrinsics = [] + img_names = [] + depth_names = [] + for i, line in enumerate(lines): + line = line.strip() + if 'intrinsics_matrix' in line: + line = line.replace('intrinsics_matrix ', '') + line = line.split(' ') + line = remove_items(line, '') + if len(line) !=9: + print('[WARN] something wrong at {}'.format(i)) + intrinsic = np.asarray(line).astype(float).reshape(3, 3) + intrinsics.extend([intrinsic, intrinsic, intrinsic, intrinsic, intrinsic, intrinsic]) + elif 'scan' in line: + line = line.split(' ') + img_names.append(line[2]) + depth_names.append(line[1]) + + line = remove_items(line, '')[3:] + if len(line) != 16: + print('[WARN] something wrong at {}'.format(i)) + extrinsic = np.asarray(line).astype(float).reshape(4, 4) + extrinsic[:3, 1] *= -1.0 # gl2cv + extrinsic[:3, 2] *= -1.0 + extrinsics.append(extrinsic) + + intrinsics = np.stack(intrinsics, axis=0) + extrinsics = np.stack(extrinsics, axis=0) + img_names = np.asarray(img_names) + + return img_names, depth_names, intrinsics, extrinsics + + + def get_intrinsics(self, frame_id): + K = self.intrinsics[frame_id] + intrinisc_cam_parameters = o3d.camera.PinholeCameraIntrinsic() + intrinisc_cam_parameters.set_intrinsics(self.image_size[0], self.image_size[1], K[0, 0], K[1, 1], K[0, 2], K[1, 2]) + return intrinisc_cam_parameters + + + def get_extrinsic(self, frame_id): + return self.extrinsics[frame_id] + + + def get_depth(self, frame_id): + depth_path = os.path.join(self.depth_dir, self.depth_names[frame_id]) + depth = cv2.imread(depth_path, -1).astype(np.uint16) + depth = depth / self.depth_scale + depth = depth.astype(np.float32) + return depth + + + def get_rgb(self, frame_id, change_color=True): + rgb = cv2.imread(os.path.join(self.rgb_dir, self.rgb_names[frame_id])) + if change_color: + rgb = cv2.cvtColor(rgb, cv2.COLOR_BGR2RGB) + return rgb + + + def get_segmentation(self, frame_id, align_with_depth=False): + frame_name = self.rgb_names[frame_id][:-4] + segmentation_path = os.path.join(self.segmentation_dir, f'{frame_name}.png') + if not os.path.exists(segmentation_path): + assert False, f"Segmentation not found: {segmentation_path}" + segmentation = cv2.imread(segmentation_path, cv2.IMREAD_UNCHANGED) + return segmentation + + + def get_frame_path(self, frame_id): + rgb_path = os.path.join(self.rgb_dir, self.rgb_names[frame_id]) + frame_name = self.rgb_names[frame_id][:-4] + segmentation_path = os.path.join(self.segmentation_dir, f'{frame_name}.png') + return rgb_path, segmentation_path + + + def get_label_features(self): + label_features_dict = np.load(f'data/text_features/matterport3d.npy', allow_pickle=True).item() + return label_features_dict + + + def get_scene_points(self): + mesh = o3d.io.read_point_cloud(self.point_cloud_path) + vertices = np.asarray(mesh.points) + return vertices + + + def get_label_id(self): + self.label2id = {} + self.id2label = {} + for label, id in zip(MATTERPORT_LABELS, MATTERPORT_IDS): + self.label2id[label] = id + self.id2label[id] = label + return self.label2id, self.id2label \ No newline at end of file diff --git a/MaskClustering/dataset/scannet.py b/MaskClustering/dataset/scannet.py new file mode 100644 index 0000000000000000000000000000000000000000..f88a5e794fc4533f2cc61a8c9a232b39d49a6f60 --- /dev/null +++ b/MaskClustering/dataset/scannet.py @@ -0,0 +1,452 @@ +import open3d as o3d +import numpy as np +import os +import cv2 +from evaluation.constants import SCANNET_LABELS, SCANNET_IDS, SCANNET18_LABELS, SCANNET18_IDS, SCANNETPP84_IDS, SCANNETPP84_LABELS, SCANNET20_LABELS, SCANNET20_IDS, ARKIT_LABELS, ARKIT_IDS + +class ScanNetDataset: + + def __init__(self, seq_name, root='data/scannet', use_templates=False) -> None: + self.seq_name = seq_name + self.use_templates = use_templates + self.root = os.path.join(root, 'processed', seq_name) + self.rgb_dir = f'{self.root}/color' + self.depth_dir = f'{self.root}/depth' + self.segmentation_dir = f'{self.root}/output/mask' + self.object_dict_dir = f'{self.root}/output/object' + self.point_cloud_path = f'{self.root}/{seq_name}_vh_clean_2.ply' + self.mesh_path = self.point_cloud_path + self.extrinsics_dir = f'{self.root}/pose' + self.intrinsic_dir = f'{self.root}/intrinsic' + self.label_features_dict = None + + self.depth_scale = 1000.0 + self.image_size = self.get_image_size() + self.depth_size = self.get_depth_shape() + + + def get_frame_list(self, stride): + image_list = os.listdir(self.rgb_dir) + image_list = sorted(image_list, key=lambda x: int(x.split('.')[0])) + + end = int(image_list[-1].split('.')[0]) + 1 + frame_id_list = [int(a.split('.')[0]) for a in image_list] + return list(frame_id_list) + + def get_image_size(self): + image_list = os.listdir(self.rgb_dir) + image_list = sorted(image_list, key=lambda x: int(x.split('.')[0])) + image_path = os.path.join(self.rgb_dir, image_list[0]) + image = cv2.imread(image_path) + return image.shape[:2][::-1] + + def get_depth_shape(self): + image_list = os.listdir(self.rgb_dir) + image_list = sorted(image_list, key=lambda x: int(x.split('.')[0])) + depth_path = os.path.join(self.depth_dir, f"{image_list[0].split('.')[0]}.png") + depth = cv2.imread(depth_path, -1) + return depth.shape[:2][::-1] + + def get_intrinsics(self, frame_id): + intrinsic_path = f'{self.intrinsic_dir}/intrinsic_depth.txt' + intrinsics = np.loadtxt(intrinsic_path) + + intrinisc_cam_parameters = o3d.camera.PinholeCameraIntrinsic() + intrinisc_cam_parameters.set_intrinsics(self.image_size[0], self.image_size[1], intrinsics[0, 0], intrinsics[1, 1], intrinsics[0, 2], intrinsics[1, 2]) + return intrinisc_cam_parameters + + + def get_extrinsic(self, frame_id): + pose_path = os.path.join(self.extrinsics_dir, str(frame_id) + '.txt') + pose = np.loadtxt(pose_path) + return pose + + + def get_depth(self, frame_id): + depth_path = os.path.join(self.depth_dir, str(frame_id) + '.png') + depth = cv2.imread(depth_path, -1) + depth = depth / self.depth_scale + depth = depth.astype(np.float32) + return depth + + + def get_rgb(self, frame_id, change_color=True): + rgb_path = os.path.join(self.rgb_dir, str(frame_id) + '.jpg') + rgb = cv2.imread(rgb_path) + + if change_color: + rgb = cv2.cvtColor(rgb, cv2.COLOR_BGR2RGB) + return rgb + + + def get_segmentation(self, frame_id, align_with_depth=False): + segmentation_path = os.path.join(self.segmentation_dir, f'{frame_id}.png') + if not os.path.exists(segmentation_path): + assert False, f"Segmentation not found: {segmentation_path}" + segmentation = cv2.imread(segmentation_path, cv2.IMREAD_UNCHANGED) + if align_with_depth: + segmentation = cv2.resize(segmentation, self.depth_size, interpolation=cv2.INTER_NEAREST) + return segmentation + + + def get_frame_path(self, frame_id): + rgb_path = os.path.join(self.rgb_dir, str(frame_id) + '.jpg') + segmentation_path = os.path.join(self.segmentation_dir, f'{frame_id}.png') + return rgb_path, segmentation_path + + + def get_label_features(self): + if self.label_features_dict is None: + if self.use_templates: + label_features_dict = np.load(f'data/text_features/scannet_templates.npy', allow_pickle=True).item() + else: + label_features_dict = np.load(f'data/text_features/scannet.npy', allow_pickle=True).item() + self.label_features_dict = label_features_dict + return self.label_features_dict + + + def get_scene_points(self): + mesh = o3d.io.read_point_cloud(self.point_cloud_path) + vertices = np.asarray(mesh.points) + return vertices + + + def get_label_id(self): + self.class_id = SCANNET_IDS + self.class_label = SCANNET_LABELS + + self.label2id = {} + self.id2label = {} + for label, id in zip(self.class_label, self.class_id): + self.label2id[label] = id + self.id2label[id] = label + + return self.label2id, self.id2label + + +class ARKitDataset(ScanNetDataset): + def __init__(self, seq_name, root='data/arkit_dust3r_posed'): + super().__init__(seq_name, root) + self.image_size = self.get_image_size() + + def get_image_size(self): + image_list = os.listdir(self.rgb_dir) + image_list = sorted(image_list, key=lambda x: int(x.split('.')[0])) + image_path = os.path.join(self.rgb_dir, image_list[0]) + image = cv2.imread(image_path) + return image.shape[:2][::-1] + + def get_frame_list(self, stride): + image_list = os.listdir(self.rgb_dir) + image_list = sorted(image_list, key=lambda x: int(x.split('.')[0])) + + end = int(image_list[-1].split('.')[0]) + 1 + frame_id_list = [a.split('.')[0] for a in image_list] + return list(frame_id_list) + + def get_label_id(self): + self.class_id = ARKIT_IDS + self.class_label = ARKIT_LABELS + + self.label2id = {} + self.id2label = {} + for label, id in zip(self.class_label, self.class_id): + self.label2id[label] = id + self.id2label[id] = label + + return self.label2id, self.id2label + + def get_label_features(self): + label_features_dict = np.load(f'data/text_features/arkit.npy', allow_pickle=True).item() + return label_features_dict + +class ITWDataset(ARKitDataset): + + def get_image_size(self): + image_list = os.listdir(self.rgb_dir) + image_list = sorted(image_list, key=lambda x: int(x.split('_')[0])) + image_path = os.path.join(self.rgb_dir, image_list[0]) + image = cv2.imread(image_path) + return image.shape[:2][::-1] + + def get_depth_shape(self): + image_list = os.listdir(self.rgb_dir) + image_list = sorted(image_list, key=lambda x: int(x.split('_')[0])) + depth_path = os.path.join(self.depth_dir, f"{image_list[0].split('.')[0]}.png") + depth = cv2.imread(depth_path, -1) + return depth.shape[:2][::-1] + + def get_frame_list(self, stride): + image_list = os.listdir(self.rgb_dir) + image_list = sorted(image_list, key=lambda x: int(x.split('_')[0])) + + frame_id_list = [a.split('.')[0] for a in image_list] + return list(frame_id_list) + + def get_label_features(self): + label_features_dict = np.load(f'{self.root}/text_features.npy', allow_pickle=True).item() + return label_features_dict + + def get_label_id(self): + text_features = self.get_label_features() + + self.class_label = list(text_features.keys()) + self.class_id = list(range(len(self.class_label))) + + self.label2id = {} + self.id2label = {} + for label, id in zip(self.class_label, self.class_id): + self.label2id[label] = id + self.id2label[id] = label + + return self.label2id, self.id2label + +class WildDataset(ARKitDataset): + def __init__(self, seq_name, root): + self.root = os.path.join(root, seq_name) + self.rgb_dir = f'{self.root}/images' + self.depth_dir = f'{self.root}/depth' + self.segmentation_dir = f'{self.root}/output/mask' + self.object_dict_dir = f'{self.root}/output/object' + self.point_cloud_path = f'{self.root}/point_cloud.ply' + self.mesh_path = self.point_cloud_path + self.extrinsics_dir = f'{self.root}/pose' + self.intrinsic_dir = f'{self.root}/intrinsic' + self.label_features_dict = None + + self.depth_scale = 1000.0 + self.image_size = self.get_depth_shape() + self.depth_size = self.get_depth_shape() + + def get_label_features(self): + label_features_dict = np.load(f'{self.root}/text_features.npy', allow_pickle=True).item() + return label_features_dict + + def get_segmentation(self, frame_id, align_with_depth=False): + segmentation_path = os.path.join(self.segmentation_dir, f'{frame_id}.png') + if not os.path.exists(segmentation_path): + assert False, f"Segmentation not found: {segmentation_path}" + segmentation = cv2.imread(segmentation_path, cv2.IMREAD_UNCHANGED) + segmentation = cv2.resize(segmentation, self.depth_size, interpolation=cv2.INTER_NEAREST) + return segmentation + def get_label_id(self): + text_features = self.get_label_features() + + self.class_label = list(text_features.keys()) + self.class_id = list(range(len(self.class_label))) + + self.label2id = {} + self.id2label = {} + for label, id in zip(self.class_label, self.class_id): + self.label2id[label] = id + self.id2label[id] = label + + return self.label2id, self.id2label + + +class ScannetPP2Dataset(ScanNetDataset): + def __init__(self, seq_name, root='data/scannetpp_dust3r_posed'): + super().__init__(seq_name, root) + self.image_size = self.get_image_size() + self.depth_size = self.get_depth_shape() + + self.point_cloud_path = f'{self.root}/{seq_name}.ply' + + def get_image_size(self): + image_list = os.listdir(self.rgb_dir) + image_list = sorted(image_list, key=lambda x: int(x.split('.')[0].split('_')[1])) + image_path = os.path.join(self.rgb_dir, image_list[0]) + image = cv2.imread(image_path) + return image.shape[:2][::-1] + + def get_depth_shape(self): + + image_list = os.listdir(self.rgb_dir) + image_list = sorted(image_list, key=lambda x: int(x.split('.')[0].split('_')[1])) + depth_path = os.path.join(self.depth_dir, f"{image_list[0].split('.')[0]}.png") + depth = cv2.imread(depth_path, -1) + return depth.shape[:2][::-1] + + def get_frame_list(self, stride): + image_list = os.listdir(self.rgb_dir) + image_list = sorted(image_list, key=lambda x: int(x.split('.')[0].split('_')[1])) + + frame_id_list = [a.split('.')[0] for a in image_list] + return list(frame_id_list) + + def get_segmentation(self, frame_id, align_with_depth=False): + segmentation_path = os.path.join(self.segmentation_dir, f'{frame_id}.png') + if not os.path.exists(segmentation_path): + assert False, f"Segmentation not found: {segmentation_path}" + segmentation = cv2.imread(segmentation_path, cv2.IMREAD_UNCHANGED) + segmentation = cv2.resize(segmentation, self.depth_size, interpolation=cv2.INTER_NEAREST) + return segmentation + + + def get_label_id(self): + self.class_id = SCANNETPP84_IDS + self.class_label = SCANNETPP84_LABELS + + self.label2id = {} + self.id2label = {} + for label, id in zip(self.class_label, self.class_id): + self.label2id[label] = id + self.id2label[id] = label + + return self.label2id, self.id2label + + def get_label_features(self): + label_features_dict = np.load(f'data/text_features/scannetpp84.npy', allow_pickle=True).item() + return label_features_dict + + def get_depth(self, frame_id): + depth_path = os.path.join(self.depth_dir, str(frame_id) + '.png') + depth = cv2.imread(depth_path, -1) + depth = depth / self.depth_scale + depth = depth.astype(np.float32) + return depth + + + def get_intrinsics(self, frame_id): + intrinsic_path = f'{self.intrinsic_dir}/intrinsic_depth.txt' + intrinsics = np.loadtxt(intrinsic_path) + + intrinisc_cam_parameters = o3d.camera.PinholeCameraIntrinsic() + intrinisc_cam_parameters.set_intrinsics(self.image_size[0], self.image_size[1], intrinsics[0, 0], intrinsics[1, 1], intrinsics[0, 2], intrinsics[1, 2]) + return intrinisc_cam_parameters + + +class ScanNet18Dataset: + + def __init__(self, seq_name, root='data/scannet') -> None: + self.seq_name = seq_name + self.root = os.path.join(root, 'processed', seq_name) + self.rgb_dir = f'{self.root}/color' + self.depth_dir = f'{self.root}/depth' + self.segmentation_dir = f'{self.root}/output/mask' + self.object_dict_dir = f'{self.root}/output/object' + self.point_cloud_path = f'{self.root}/{seq_name}.ply' + self.mesh_path = self.point_cloud_path + self.extrinsics_dir = f'{self.root}/pose' + self.intrinsic_dir = f'{self.root}/intrinsic' + + self.depth_scale = 1000.0 + self.image_size = self.get_image_size() + self.depth_size = self.get_depth_shape() + + + def get_frame_list(self, stride): + image_list = os.listdir(self.rgb_dir) + image_list = sorted(image_list, key=lambda x: int(x.split('.')[0])) + + end = int(image_list[-1].split('.')[0]) + 1 + frame_id_list = [a.split('.')[0] for a in image_list] + return list(frame_id_list) + + def get_image_size(self): + image_list = os.listdir(self.rgb_dir) + image_list = sorted(image_list, key=lambda x: int(x.split('.')[0])) + image_path = os.path.join(self.rgb_dir, image_list[0]) + image = cv2.imread(image_path) + return image.shape[:2][::-1] + + def get_depth_shape(self): + image_list = os.listdir(self.rgb_dir) + image_list = sorted(image_list, key=lambda x: int(x.split('.')[0])) + depth_path = os.path.join(self.depth_dir, f"{image_list[0].split('.')[0]}.png") + depth = cv2.imread(depth_path, -1) + return depth.shape[:2][::-1] + + def get_intrinsics(self, frame_id): + intrinsic_path = f'{self.intrinsic_dir}/intrinsic_depth.txt' + intrinsics = np.loadtxt(intrinsic_path) + + intrinisc_cam_parameters = o3d.camera.PinholeCameraIntrinsic() + intrinisc_cam_parameters.set_intrinsics(self.image_size[0], self.image_size[1], intrinsics[0, 0], intrinsics[1, 1], intrinsics[0, 2], intrinsics[1, 2]) + return intrinisc_cam_parameters + + + def get_extrinsic(self, frame_id): + pose_path = os.path.join(self.extrinsics_dir, str(frame_id) + '.txt') + pose = np.loadtxt(pose_path) + return pose + + + def get_depth(self, frame_id): + depth_path = os.path.join(self.depth_dir, str(frame_id) + '.png') + depth = cv2.imread(depth_path, -1) + depth = depth / self.depth_scale + depth = depth.astype(np.float32) + return depth + + + def get_rgb(self, frame_id, change_color=True): + rgb_path = os.path.join(self.rgb_dir, str(frame_id) + '.jpg') + rgb = cv2.imread(rgb_path) + + if change_color: + rgb = cv2.cvtColor(rgb, cv2.COLOR_BGR2RGB) + return rgb + + + def get_segmentation(self, frame_id, align_with_depth=False): + segmentation_path = os.path.join(self.segmentation_dir, f'{frame_id}.png') + if not os.path.exists(segmentation_path): + assert False, f"Segmentation not found: {segmentation_path}" + segmentation = cv2.imread(segmentation_path, cv2.IMREAD_UNCHANGED) + segmentation = cv2.resize(segmentation, self.depth_size, interpolation=cv2.INTER_NEAREST) + return segmentation + + + def get_frame_path(self, frame_id): + rgb_path = os.path.join(self.rgb_dir, str(frame_id) + '.jpg') + segmentation_path = os.path.join(self.segmentation_dir, f'{frame_id}.png') + return rgb_path, segmentation_path + + + def get_label_features(self): + label_features_dict = np.load(f'data/text_features/scannet18.npy', allow_pickle=True).item() + return label_features_dict + + + def get_scene_points(self): + mesh = o3d.io.read_point_cloud(self.point_cloud_path) + vertices = np.asarray(mesh.points) + return vertices + + + def get_label_id(self): + self.class_id = SCANNET18_IDS + self.class_label = SCANNET18_LABELS + + self.label2id = {} + self.id2label = {} + for label, id in zip(self.class_label, self.class_id): + self.label2id[label] = id + self.id2label[id] = label + + return self.label2id, self.id2label + + +class ScanNet20Dataset(ScanNet18Dataset): + + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + self.point_cloud_path = f'{self.root}/{self.seq_name}_vh_clean_2.ply' + + def get_label_features(self): + label_features_dict = np.load(f'/home/jovyan/users/lemeshko/Indoor/MaskClustering/data/text_features/scannet20.npy', allow_pickle=True).item() + return label_features_dict + + + def get_label_id(self): + self.class_id = SCANNET20_IDS + self.class_label = SCANNET20_LABELS + + self.label2id = {} + self.id2label = {} + for label, id in zip(self.class_label, self.class_id): + self.label2id[label] = id + self.id2label[id] = label + + return self.label2id, self.id2label \ No newline at end of file diff --git a/MaskClustering/dataset/scannetpp.py b/MaskClustering/dataset/scannetpp.py new file mode 100644 index 0000000000000000000000000000000000000000..36f90f88a6cb25f946f8c8facbb6f79034e29764 --- /dev/null +++ b/MaskClustering/dataset/scannetpp.py @@ -0,0 +1,217 @@ +import open3d as o3d +import numpy as np +import os +import cv2 +import collections +from evaluation.constants import SCANNETPP_LABELS, SCANNETPP_IDS +import torch + +BaseImage = collections.namedtuple( + "Image", ["id", "qvec", "tvec", "camera_id", "name", "xys", "point3D_ids"]) +BaseCamera = collections.namedtuple( + "Camera", ["id", "model", "width", "height", "params"]) + + +def qvec2rotmat(qvec): + return np.array([ + [1 - 2 * qvec[2]**2 - 2 * qvec[3]**2, + 2 * qvec[1] * qvec[2] - 2 * qvec[0] * qvec[3], + 2 * qvec[3] * qvec[1] + 2 * qvec[0] * qvec[2]], + [2 * qvec[1] * qvec[2] + 2 * qvec[0] * qvec[3], + 1 - 2 * qvec[1]**2 - 2 * qvec[3]**2, + 2 * qvec[2] * qvec[3] - 2 * qvec[0] * qvec[1]], + [2 * qvec[3] * qvec[1] - 2 * qvec[0] * qvec[2], + 2 * qvec[2] * qvec[3] + 2 * qvec[0] * qvec[1], + 1 - 2 * qvec[1]**2 - 2 * qvec[2]**2]]) + + +class Image(BaseImage): + def qvec2rotmat(self): + return qvec2rotmat(self.qvec) + + @property + def world_to_camera(self) -> np.ndarray: + R = qvec2rotmat(self.qvec) + t = self.tvec + world2cam = np.eye(4) + world2cam[:3, :3] = R + world2cam[:3, 3] = t + return world2cam + + +class Camera(BaseCamera): + @property + def K(self): + K = np.eye(3) + if self.model == "SIMPLE_PINHOLE" or self.model == "SIMPLE_RADIAL" or self.model == "RADIAL" or self.model == "SIMPLE_RADIAL_FISHEYE" or self.model == "RADIAL_FISHEYE": + K[0, 0] = self.params[0] + K[1, 1] = self.params[0] + K[0, 2] = self.params[1] + K[1, 2] = self.params[2] + elif self.model == "PINHOLE" or self.model == "OPENCV" or self.model == "OPENCV_FISHEYE" or self.model == "FULL_OPENCV" or self.model == "FOV" or self.model == "THIN_PRISM_FISHEYE": + K[0, 0] = self.params[0] + K[1, 1] = self.params[1] + K[0, 2] = self.params[2] + K[1, 2] = self.params[3] + else: + raise NotImplementedError + return K + + +def read_images_text(path): + images = {} + with open(path, "r") as fid: + while True: + line = fid.readline() + if not line: + break + line = line.strip() + if len(line) > 0 and line[0] != "#": + elems = line.split() + image_id = int(elems[0]) + qvec = np.array(tuple(map(float, elems[1:5]))) + tvec = np.array(tuple(map(float, elems[5:8]))) + camera_id = int(elems[8]) + image_name = elems[9] + elems = fid.readline().split() + xys = np.column_stack([tuple(map(float, elems[0::3])), + tuple(map(float, elems[1::3]))]) + point3D_ids = np.array(tuple(map(int, elems[2::3]))) + images[image_id] = Image( + id=image_id, qvec=qvec, tvec=tvec, + camera_id=camera_id, name=image_name, + xys=xys, point3D_ids=point3D_ids) + return images + + +def read_cameras_text(path): + """ + see: src/base/reconstruction.cc + void Reconstruction::WriteCamerasText(const std::string& path) + void Reconstruction::ReadCamerasText(const std::string& path) + """ + cameras = {} + with open(path, "r") as fid: + while True: + line = fid.readline() + if not line: + break + line = line.strip() + if len(line) > 0 and line[0] != "#": + elems = line.split() + camera_id = int(elems[0]) + model = elems[1] + width = int(elems[2]) + height = int(elems[3]) + params = np.array(tuple(map(float, elems[4:]))) + cameras[camera_id] = Camera(id=camera_id, model=model, + width=width, height=height, + params=params) + return cameras + + +class ScanNetPPDataset: + + def __init__(self, seq_name) -> None: + self.seq_name = seq_name + self.root = f'./data/scannetpp/data/{seq_name}' + self.rgb_dir = f'{self.root}/iphone/rgb' + self.depth_dir = f'{self.root}/iphone/render_depth' + self.segmentation_dir = f'{self.root}/output/mask' + self.object_dict_dir = f'{self.root}/output/object' + self.point_cloud_path = f'./data/scannetpp/pcld_0.25/{seq_name}.pth' + self.load_meta_data() + + self.depth_scale = 1000.0 + self.image_size = (1920, 1440) + + + def load_meta_data(self): + self.frame_id_list = [] + + cameras = read_cameras_text(os.path.join(self.root, 'iphone/colmap', "cameras.txt")) + images = read_images_text(os.path.join(self.root, 'iphone/colmap', "images.txt")) + camera = next(iter(cameras.values())) + fx, fy, cx, cy = camera.params[:4] + intrinsics = {} + extrinsics = {} + + for _, image in (images.items()): + image_id = int(image.name.split('.')[0].split('_')[1]) + self.frame_id_list.append(image_id) + world_to_camera = image.world_to_camera + extrinsics[image_id] = np.linalg.inv(world_to_camera) + intrinsics[image_id] = np.array([[fx, 0, cx], [0, fy, cy], [0, 0, 1]]) + + self.extrinsics = extrinsics + self.intrinsics = intrinsics + + + def get_frame_list(self, stride): + return self.frame_id_list[::stride] + + + def get_intrinsics(self, frame_id): + intrinsic_matrix = self.intrinsics[frame_id] + + intrinisc_cam_parameters = o3d.camera.PinholeCameraIntrinsic() + intrinisc_cam_parameters.set_intrinsics(self.image_size[0], self.image_size[1], intrinsic_matrix[0, 0], intrinsic_matrix[1, 1], intrinsic_matrix[0, 2], intrinsic_matrix[1, 2]) + return intrinisc_cam_parameters + + + def get_extrinsic(self, frame_id): + return self.extrinsics[frame_id] + + + def get_depth(self, frame_id): + depth_path = os.path.join(self.depth_dir, 'frame_%06d.png' % frame_id) + depth = cv2.imread(depth_path, -1) + depth = depth / self.depth_scale + depth = depth.astype(np.float32) + return depth + + + def get_rgb(self, frame_id, change_color=True): + rgb_path = os.path.join(self.rgb_dir, 'frame_%06d.jpg' % frame_id) + rgb = cv2.imread(rgb_path) + if change_color: + rgb = cv2.cvtColor(rgb, cv2.COLOR_BGR2RGB) + return rgb + + + def get_segmentation(self, frame_id, align_with_depth=False): + segmentation_path = os.path.join(self.segmentation_dir, 'frame_%06d.png' % frame_id) + if not os.path.exists(segmentation_path): + assert False, f"Segmentation not found: {segmentation_path}" + segmentation = cv2.imread(segmentation_path, cv2.IMREAD_UNCHANGED) + return segmentation + + + def get_frame_path(self, frame_id): + rgb_path = os.path.join(self.rgb_dir, 'frame_%06d.jpg' % frame_id) + segmentation_path = os.path.join(self.segmentation_dir, 'frame_%06d.png' % frame_id) + return rgb_path, segmentation_path + + + def get_label_features(self): + label_features_dict = np.load(f'data/text_features/scannetpp.npy', allow_pickle=True).item() + return label_features_dict + + + def get_scene_points(self): + data = torch.load(self.point_cloud_path) + points = np.asarray(data['sampled_coords']) + return points + + + def get_label_id(self): + self.class_id = SCANNETPP_IDS + self.class_label = SCANNETPP_LABELS + + self.label2id = {} + self.id2label = {} + for label, id in zip(self.class_label, self.class_id): + self.label2id[label] = id + self.id2label[id] = label + + return self.label2id, self.id2label \ No newline at end of file diff --git a/MaskClustering/dense_masks.py b/MaskClustering/dense_masks.py new file mode 100644 index 0000000000000000000000000000000000000000..7c83f3ec2f91f116bc20cb750b30b1b2633db234 --- /dev/null +++ b/MaskClustering/dense_masks.py @@ -0,0 +1,91 @@ +import numpy as np +import os +import cv2 +from pathlib import Path +import trimesh as tm +from sklearn.neighbors import KDTree +from tqdm import tqdm +from tqdm.contrib.concurrent import thread_map +from sklearn.cluster import DBSCAN + + +def load_scan(pcd_path): + pcd_data = np.fromfile(pcd_path, dtype=np.float32).reshape(-1, 6)[:, :3] + return pcd_data + +def process_scene(data): + scene_id, exp_name = data + pred_path = Path(f"data/prediction/scannet/baseline_scannet200/{scene_id}.npz") + out_path = Path(f"data/prediction/scannet/{exp_name}/{scene_id}.npz") + base_path = Path(f"/home/jovyan/users/lemeshko/scripts/gsam_result/yolo/{scene_id}") + source_path = Path(f"/home/jovyan/users/kolodiazhnyi/data/scannet/posed_images/{scene_id}") + scan_path = Path(f"/home/jovyan/users/bulat/workspace/3drec/Indoor/OKNO/data/scannet200/points/{scene_id}.bin") + info_path = base_path / "infos.npy" + + # if out_path.exists(): + # return + vertices = load_scan(scan_path) + kd = KDTree(vertices) + max_dist = 0.05 + + base_data = np.load(pred_path, allow_pickle=True) + + total_points_masks = base_data['pred_masks'].T + + for i, mask in enumerate(total_points_masks): + mask = mask.astype(bool) + points = vertices[mask] + dists, inds = kd.query(points, k=5) + dists = dists.flatten() + inds = inds.flatten() + + dist_mask = dists < max_dist + inds = inds[dist_mask] + mask[inds] = True + total_points_masks[i] = mask + + for i, mask in enumerate(total_points_masks): + mask = mask.astype(bool) + points = vertices[mask] + db = DBSCAN(eps=0.3, min_samples=10) + labels = db.fit_predict(points) + + # labels = db.labels_ + n_clusters = len(set(labels)) - (1 if -1 in labels else 0) + biggest_cluster_ind = np.argmax(np.unique(labels, return_counts=True)[1]) + res_mask = (labels == biggest_cluster_ind) & (labels != -1) + # print(f"{labels.shape} -> {res_mask.sum()}") + new_mask = np.zeros_like(mask) + new_mask[mask] = res_mask + total_points_masks[i] = new_mask + + new_data = { + k: v for k, v in base_data.items() + } + new_data['pred_masks'] = total_points_masks.T + + out_path.parent.mkdir(parents=True, exist_ok=True) + # vs = [] + # cs = [] + # for i in range(new_data['pred_masks'].shape[1]): + # os.makedirs(f"pred_masks", exist_ok=True) + # v = vertices[new_data['pred_masks'][:, i]] + # c = np.random.rand(3) + # c = np.repeat(c[np.newaxis, :], len(v), axis=0) + # vs.append(v) + # cs.append(c) + # tm.PointCloud(np.concatenate(vs, axis=0), colors=np.concatenate(cs, axis=0)).export(f"pred_masks/{scene_id}_mask.ply") + + print("uniques", np.unique(new_data['pred_masks'].sum(1)), [[k, v.shape] for k, v in new_data.items()]) + np.savez(out_path, **new_data) + + + +if __name__ == "__main__": + exp_name = "dense_masks_cluster_filtering" + scenes = np.loadtxt("/home/jovyan/users/bulat/workspace/3drec/Indoor/MaskClustering/splits/scannet.txt", dtype=str) + data = [(scene, exp_name) for scene in scenes] + total_points_masks = thread_map(process_scene, data, chunksize=20) + + + diff --git a/MaskClustering/evaluation/__init__.py b/MaskClustering/evaluation/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8701dc7561b03c446e4cf79890f4e35a39ccf1ef --- /dev/null +++ b/MaskClustering/evaluation/__init__.py @@ -0,0 +1 @@ +# Инициализация модуля diff --git a/MaskClustering/evaluation/constants.py b/MaskClustering/evaluation/constants.py new file mode 100644 index 0000000000000000000000000000000000000000..c79e90f82c223a4170c0a45ca83f5b3f352d998a --- /dev/null +++ b/MaskClustering/evaluation/constants.py @@ -0,0 +1,78 @@ +MATTERPORT_LABELS = ('door', 'picture', 'window', 'chair', 'pillow', 'lamp', + 'cabinet', 'curtain', 'table', 'plant', 'mirror', 'towel', 'sink', 'shelves', 'sofa', + 'bed', 'night stand', 'toilet', 'column', 'banister', 'stairs', 'stool', 'vase', + 'television', 'pot', 'desk', 'box', 'coffee table', 'counter', 'bench', 'garbage bin', + 'fireplace', 'clothes', 'bathtub', 'book', 'air vent', 'faucet', 'photo', 'toilet paper', + 'fan', 'railing', 'sculpture', 'dresser', 'rug', 'ottoman', 'bottle', 'refridgerator', + 'bookshelf', 'wardrobe', 'pipe', 'monitor', 'stand', 'drawer', 'container', 'light switch', + 'purse', 'door way', 'basket', 'chandelier', 'oven', 'clock', 'stove', 'washing machine', + 'shower curtain', 'fire alarm', 'bin', 'chest', 'microwave', 'blinds', 'bowl', 'tissue box', + 'plate', 'tv stand', 'shoe', 'heater', 'headboard', 'bucket', 'candle', 'flower pot', + 'speaker', 'furniture', 'sign', 'air conditioner', 'fire extinguisher', 'curtain rod', + 'floor mat', 'printer', 'telephone', 'blanket', 'handle', 'shower head', 'soap', 'keyboard', + 'thermostat', 'radiator', 'kitchen island', 'paper towel', 'sheet', 'glass', 'dishwasher', + 'cup', 'ladder', 'garage door', 'hat', 'exit sign', 'piano', 'board', 'rope', 'ball', + 'excercise equipment', 'hanger', 'candlestick', 'light', 'scale', 'bag', 'laptop', 'treadmill', + 'guitar', 'display case', 'toilet paper holder', 'bar', 'tray', 'urn', 'decorative plate', 'pool table', + 'jacket', 'bottle of soap', 'water cooler', 'utensil', 'tea pot', 'stuffed animal', 'paper towel dispenser', + 'lamp shade', 'car', 'toilet brush', 'doll', 'drum', 'whiteboard', 'range hood', 'candelabra', 'toy', + 'foot rest', 'soap dish', 'placemat', 'cleaner', 'computer', 'knob', 'paper', 'projector', 'coat hanger', + 'case', 'pan', 'luggage', 'trinket', 'chimney', 'person', 'alarm') + +MATTERPORT_IDS = [28, 64, 59, 5, 119, 144, 3, 89, 19, 82, 122, 135, 24, 42, 83, 157, 158, 124, 94, 453, + 215, 150, 78, 172, 16, 36, 26, 356, 7, 204, 12, 372, 141, 136, 1, 25, 9, 508, 139, 74, 497, 294, + 169, 130, 359, 2, 17, 88, 772, 41, 49, 50, 174, 140, 301, 181, 609, 39, 342, 238, 56, 242, 278, + 123, 338, 307, 344, 13, 80, 22, 138, 233, 291, 149, 111, 161, 427, 137, 146, 54, 524, 208, 79, + 10, 582, 143, 66, 32, 312, 758, 650, 133, 47, 110, 236, 456, 113, 559, 612, 8, 35, 48, 850, 193, + 86, 298, 408, 560, 60, 457, 211, 148, 62, 639, 55, 37, 458, 300, 540, 647, 51, 179, 151, 383, 515, + 324, 502, 509, 267, 678, 177, 14, 859, 530, 630, 99, 145, 45, 380, 605, 389, 163, 638, 154, 548, + 46, 652, 15, 90, 400, 851, 589, 783, 844, 702, 331, 525] + +SCANNET_LABELS = ['chair', 'table', 'door', 'couch', 'cabinet', 'shelf', 'desk', 'office chair', 'bed', 'pillow', 'sink', 'picture', 'window', 'toilet', 'bookshelf', 'monitor', 'curtain', 'book', 'armchair', 'coffee table', 'box', +'refrigerator', 'lamp', 'kitchen cabinet', 'towel', 'clothes', 'tv', 'nightstand', 'counter', 'dresser', 'stool', 'cushion', 'plant', 'ceiling', 'bathtub', 'end table', 'dining table', 'keyboard', 'bag', 'backpack', 'toilet paper', +'printer', 'tv stand', 'whiteboard', 'blanket', 'shower curtain', 'trash can', 'closet', 'stairs', 'microwave', 'stove', 'shoe', 'computer tower', 'bottle', 'bin', 'ottoman', 'bench', 'board', 'washing machine', 'mirror', 'copier', +'basket', 'sofa chair', 'file cabinet', 'fan', 'laptop', 'shower', 'paper', 'person', 'paper towel dispenser', 'oven', 'blinds', 'rack', 'plate', 'blackboard', 'piano', 'suitcase', 'rail', 'radiator', 'recycling bin', 'container', +'wardrobe', 'soap dispenser', 'telephone', 'bucket', 'clock', 'stand', 'light', 'laundry basket', 'pipe', 'clothes dryer', 'guitar', 'toilet paper holder', 'seat', 'speaker', 'column', 'bicycle', 'ladder', 'bathroom stall', 'shower wall', +'cup', 'jacket', 'storage bin', 'coffee maker', 'dishwasher', 'paper towel roll', 'machine', 'mat', 'windowsill', 'bar', 'toaster', 'bulletin board', 'ironing board', 'fireplace', 'soap dish', 'kitchen counter', 'doorframe', +'toilet paper dispenser', 'mini fridge', 'fire extinguisher', 'ball', 'hat', 'shower curtain rod', 'water cooler', 'paper cutter', 'tray', 'shower door', 'pillar', 'ledge', 'toaster oven', 'mouse', 'toilet seat cover dispenser', +'furniture', 'cart', 'storage container', 'scale', 'tissue box', 'light switch', 'crate', 'power outlet', 'decoration', 'sign', 'projector', 'closet door', 'vacuum cleaner', 'candle', 'plunger', 'stuffed animal', 'headphones', 'dish rack', +'broom', 'guitar case', 'range hood', 'dustpan', 'hair dryer', 'water bottle', 'handicap bar', 'purse', 'vent', 'shower floor', 'water pitcher', 'mailbox', 'bowl', 'paper bag', 'alarm clock', 'music stand', 'projector screen', 'divider', +'laundry detergent', 'bathroom counter', 'object', 'bathroom vanity', 'closet wall', 'laundry hamper', 'bathroom stall door', 'ceiling light', 'trash bin', 'dumbbell', 'stair rail', 'tube', 'bathroom cabinet', 'cd case', 'closet rod', +'coffee kettle', 'structure', 'shower head', 'keyboard piano', 'case of water bottles', 'coat rack', 'storage organizer', 'folded chair', 'fire alarm', 'power strip', 'calendar', 'poster', 'potted plant', 'luggage', 'mattress'] + +SCANNET_IDS = [2, 4, 5, 6, 7, 8, 9, 10, 11, 13, 14, 15, 16, 17, 18, 19, 21, 22, 23, 24, 26, 27, 28, 29, 31, 32, 33, 34, 35, 36, 38, 39, 40, 41, 42, 44, 45, 46, 47, 48, 49, 50, 51, 52, 54, 55, 56, 57, 58, 59, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, +72, 73, 74, 75, 76, 77, 78, 79, 80, 82, 84, 86, 87, 88, 89, 90, 93, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 110, 112, 115, 116, 118, 120, 121, 122, 125, 128, 130, 131, 132, 134, 136, 138, 139, 140, 141, 145, 148, 154, +155, 156, 157, 159, 161, 163, 165, 166, 168, 169, 170, 177, 180, 185, 188, 191, 193, 195, 202, 208, 213, 214, 221, 229, 230, 232, 233, 242, 250, 261, 264, 276, 283, 286, 300, 304, 312, 323, 325, 331, 342, 356, 370, 392, 395, 399, 408, 417, +488, 540, 562, 570, 572, 581, 609, 748, 776, 1156, 1163, 1164, 1165, 1166, 1167, 1168, 1169, 1170, 1171, 1172, 1173, 1174, 1175, 1176, 1178, 1179, 1180, 1181, 1182, 1183, 1184, 1185, 1186, 1187, 1188, 1189, 1190, 1191] + + +SCANNET18_LABELS = ['cabinet', 'bed', 'chair', 'sofa', 'table', 'door', 'window', + 'bookshelf', 'picture', 'counter', 'desk', 'curtain', 'refrigerator', + 'showercurtrain', 'toilet', 'sink', 'bathtub', 'garbagebin'] + +SCANNET18_IDS = [3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 14, 16, 24, 28, 33, 34, 36, 39] + +SCANNETPP_LABELS = ['door', 'table', 'cabinet', 'ceiling lamp', 'curtain', 'chair', 'blinds', 'storage cabinet', 'bookshelf', 'office chair', 'window', 'whiteboard', 'ceiling light', 'monitor', 'shelf', 'object', 'window frame', 'pipe', 'structure', 'box', 'heater', 'kitchen cabinet', 'storage rack', 'sofa', 'bed', 'shower wall', 'doorframe', 'door frame', 'roof', 'wardrobe', 'pillar', 'plant', 'blanket', 'machine', 'windowsill', 'linked retractable seats', 'window sill', 'cardboard box', 'tv', 'books', 'desk', 'computer tower', 'kitchen counter', 'trash can', 'trash bin', 'jacket', 'electrical duct', 'blackboard', 'cable tray', 'air duct', 'sink', 'carpet', 'bag', 'counter', 'refrigerator', 'picture', 'pillow', 'cupboard', 'window blind', 'towel', 'beam', 'office table', 'stool', 'suitcase', 'backpack', 'bathtub', 'rug', 'keyboard', 'rack', 'gym mat', 'toilet', 'suspended ceiling', 'shower floor', 'clothes', 'pipe storage rack', 'air conditioner', 'fume hood', 'printer', 'blind', 'poster', 'experiment bench', 'electrical control panel', 'shower curtain', 'windowframe', 'book', 'ceiling beam', 'painting', 'paper', 'ladder', 'laboratory bench', 'bench', 'milling machine', 'microwave', 'partition', 'board', 'office cabinet', 'rolling cart', 'laboratory cabinet', 'crate', 'raised floor', 'electrical panel', 'mattress', 'bottle', 'pedestal fan', 'sofa chair', 'headboard', 'fridge', 'bucket', 'kitchen unit', 'beanbag', 'oven', 'cushion', 'power socket', 'office desk', 'whiteboards', 'lab equipment', 'shoes', 'work bench', 'file cabinet', 'mirror', 'basket', 'beverage crate', 'washing machine', 'shoe rack', 'hydraulic press', 'photocopy machine', 'telephone', 'lab machine', 'sliding door', 'tv stand', 'objects', 'couch', 'coat', 'open cabinet', 'scientific equipment', 'coffee table', 'garage door', 'bin', 'radiator', 'standing lamp', 'stove', 'roller blinds', 'fume cupboard', 'pc', 'stairs', 'medical appliance', 'closet', 'trolley', 'file folder', 'projector', 'cloth', 'conference table', 'cardboard', 'blind rail', 'dishwasher', 'room divider', 'copier', 'ventilation pipe', 'bathroom cabinet', 'laptop', 'electrical box', 'arm chair', 'bar counter', 'stage', 'ceiling ventilator', 'lounge chair', 'plant pot', 'bathroom stall', 'pinboard', 'comforter', '3d printer', 'steel beam', 'projector screen', 'electric duct', 'cart', 'air pipe', 'training equipment', 'floor mounted air conditioner', 'tile wall', 'glass wall', 'exhaust fan', 'vacuum cleaner', 'laundry basket', 'nightstand', 'armchair', 'drying rack', 'indoor crane', 'storage trolley', 'dresser', 'l-shaped sofa', 'coat hanger', 'dining chair', 'office visitor chair', 'interactive board', 'hose', 'light switch', 'shower ceiling', 'coffee machine', 'cables', 'floor lamp', 'fan', 'wire tray', 'compressor', 'laboratory equipment', 'dining table', 'speaker', 'climbing wall', 'light', 'paper towel dispenser', 'coat rack', 'table lamp', 'frame', 'duvet', 'fire extinguisher', 'range hood', 'high table', 'backdrop', 'ventilation duct', 'seat', 'tablecloth', 'electrical cabinet', 'ping pong table', 'bathroom floor', 'ceiling pipe', 'bedside table', 'coffee maker', 'computer desk', 'urinal', 'loft bed', 'air vent', 'chairs', 'bedsheet', 'television', 'lamp', 'rolling chair', 'wall cabinet', 'book shelf', 'brick wall', 'treadmill', 'vent', 'shirt', 'canopy bed', 'clothes hanger', 'kettle', 'shoe', 'high stool', 'tripod', 'bar stool', 'exhaust duct', 'wooden plank', 'squat rack', 'cubicle door', 'folding screen', 'kitchen sink', 'container', 'bottles', 'ottoman', 'bicycle', 'staircase railing', 'overhead projector', 'surfboard', 'folder', 'power strip', 'high bench', 'wall beam', 'pallet cage', 'interactive whiteboard', 'floor sofa', 'duct', 'flat panel display', 'wooden frame', 'folding room divider', 'cable', 'mug', 'rolling table', 'locker', 'standing banner', 'decoration', 'clothes drying rack', 'foosball table', 'standing poster', 'bath tub', 'yoga mat', 'microscope', 'paper bag', 'mouse', 'umbrella', 'medical machine', 'smoke detector', 'cup', 'cutting board', 'console', 'drum', 'bathroom counter', 'toilet paper', 'robot car', 'exhaust pipe', 'bath cabinet', 'whiteboard stand', 'notice board', 'paper towel', 'crates', 'bed frame', 'bathroom mat', 'shower partition', 'cloth hangers', 'clothes cabinet', 'tv screen', 'babyfoot table', 'rolling curtain', 'coat stand', 'kitchen towel', 'plank', 'side table', 'storage shelf', 'mat', 'shower', 'white board', 'information board', 'backsplash', 'guitar', 'cloth rack', 'ceiling vent', 'partition wall', 'kitchen shelf', 'banner', 'file binder', 'cleaning trolley', 'racing simulator', 'workbench', 'pot', 'ac system', 'power panel', 'desk lamp', 'broom', 'cpu', 'fitted wardrobe', 'tote bag', 'plumbing pipe', 'slippers', 'blackboard frame', 'magazine', 'hose pipe', 'rolled paper', 'sweater', 'clock', 'tray', 'desk fan', 'vaccum cleaner', 'projection curtain', 'freezer display counter', 'pan', 'vase', 'glass', 'folders', 'tap', 'wall lamp', 'plate', 'laptop stand', 'small cabinet', 'file organizer', 'clothes dryer', 'wall painting', 'curtain rail', 'wheelchair', 'bottle crate', 'sheet', 'folding sofa', 'shower pan', 'plastic case', 'christmas tree', 'piano', 'ottoman chair', 'jar', 'foldable closet', 'notebook', 'calendar', 'janitor cart', 'storage box', 'rolling blinds', 'bathroom shelf', 'soap dispenser', 'binder', 'copy machine', 'rice cooker', 'gym mattress', 'car door', 'table football', 'bowl', 'light panel', 'tissue box', 'bedframe', 'wall hanging', 'jug', 'skylight', 'ceiling fan', 'dish rack', 'shelving cart', 'instant pot', 'whiteboard eraser', 'floor mat', 'socket', 'mini fridge', 'wall clock', 'boots', 'barbecue grill', 'paper shredder', 'file rack', 'floor scrubber', 'metal board', 'water heater', 'tool rack', 'recliner', 'barber chair', 'ventilator', 'trolley table', 'standing fan', 'water filter', 'shoes holder', 'vr setup', 'trashcan', 'bike', 'lab materials', 'wooden pallet', 'dustbin', 'curtain rod', 'reflection', 'toilet brush', 'exercise ball', 'air purifier', 'kitchen back splash', 'paper rack', 'toolbox', 'monitor cover', 'file', 'surfsuit', 'night stand', 'paper organizer', 'serving trolley', 'phone', 'canvas', 'camping bed', 'tower pc', 'cylinder', 'magazine stand', 'toy', 'slipper', 'air conditioning', 'hanger', 'vertical blinds', 'desk organizer', 'guitar bag', 'spray bottle', 'suit cover', 'toaster', 'spotlight', 'machine container', 'foot rest', 'shopping trolley', 'decoration piece', 'control panel', 'multifunction printer', 'jerry can', 'window head', 'cooker hood', 'basin', 'panel', 'papasan chair', 'tv mount', 'toilet seat', 'shopping bag', 'photocopier', 'tube', 'studio light', 'stuffed toy', 'cord cover', 'power cabinet', 'filer organizer', 'garage shelf', 'luggage', 'gym bag', 'exhaust hood', 'microwave oven', 'floor cushion', 'easy chair', 'bar table', 'shoe cabinet', 'paper tray', 'lab coat', 'toilet paper dispenser', 'kitchen storage rack', 'equipment', 'computer table', 'mouse pad', 'drawer', 'headphones', 'bathroom sink', 'outlet', 'toaster oven', 'tv table', 'bedside cabinet', 'rolling trolley', 'step stool', 'trousers', 'bathroom rack', 'shelf trolley', 'glass shelf', 'fabric', 'lab fridge', 'work station', 'barrel', 'mop', 'deck chair', 'bath counter', 'helmet', 'standing clothes hanger', 'garbage bin', 'study table', 'air fryer', 'plastic bag', 'oven range', 'headphone', 'kitchen counter top', 'clothes rack', 'wall unit', 'grab bar', 'flipchart', 'scarf', 'labcoat', 'hat', 'bedside lamp', 'sewing machine table', 'shower head', 'switchboard cabinet', 'flip paper', 'storage container', 'canister', 'wall board', 'shower rug', 'plastic box', 'stovetop', 'information stand', 'footstool', 'pack', 'push cart', 'table cloth', 'celing lamp', 'cupoard', 'jeans', 'smoke alarm', 'bath mat', 'softbox', 'whiteboard mount', 'paper cutter', 'cable raceway', 'water kettle', 'pelican case', 'towel rack', 'rolling shelf cart', 'built-in shelf', 'equipment cover', 'television stand', 'sheets', 'small dresser', 'light stand', 'beach umbrella', 'faucet', 'bagpack', 'dumbbell', 'water dispenser', 'medicine cabinet', 'tv console', 'mirror frame', 'chandelier', 'pen holder', 'messenger bag', 'ball', 'glass bottle', 'softbox light', 'gym ball', 'briefcase', 'plastic bottle', 'monitor stand', 'human skeleton', 'podium', 'wall strip', 'tablet', 'bedside shelf', 'headrail', 'sink counter', 'doormat', 'baseboard', 'bulletin board', 'electric hob', 'bean bag', 'high pressure cylinder', 'portable fan', 'flush button', 'wooden post', 'lectern', 'curtain frame', 'computer monitor', 'folding chair', 'tabletop', 'led ceiling fan', 'high chair', 'grill', 'metal rack', 'air conditioner tower', 'sliding door frame', 'cable rack', 'bench press', 'ironing board', 'wooden palette', 'kitchenware', 'blind rails', 'plastic container', 'weighing scale', 'headset', 'tree trunk', 'shower screen', 'wall shelf', 'watering can', 'tool box', 'bed sheet', 'glass pane', 'tower fan', 'switch', 'notice', 'sack', 'table mat', 'flower pot', 'dog bed', 'laundry hanger', 'mobile tv stand', 'file holder', 'floor couch', 'tv trolley', 'chopping board', 'centrifuge', 'tubelight', 'bedpost', 'step', 'center table', 'upholstered bench', 'sink pipe', 'door mat', 'storage bin', 'towel radiator', 'shower tray', 'electronic appliance', 'boiler', 'food container', 'cable pathway', 'carboard box', 'metal sheet', 'hand bag', 'sign', 'laundry rack', 'screen', 'cardbox', 'fireplace surround', 'boot', 'envelope', 'carton', 'tool organizer', 'paper roll', 'water bottle', 'shoe changing stool', 'balcony door', 'espresso machine', 'water pipe', 'recesssed shelf', 'drum set', 'skiboard', 'speaker stand', 'kitchen wall', 'suit', 'photo', 'globe', 'spice rack', 'delivery bag', 'router', 'rolling blind', 'easel', 'shower cubicle', 'dish drainer', 'doorway', 'folded table', 'pants', 'computer', 'stuffed animal', 'office chair', 'cable conduit', 'picture frame', 'shoe stool', 'recessed shelve', 'toilet paper holder', 'panelboard', 'stapler', 'skateboard', 'workshop tool', 'projector holder', 'flag', 'chemical canister', 'web cam', 'hoodie', 'towel heater', 'towel warmer', 'shower curtain rod', 'shower faucet', 'shower door', 'laboratory power supply', 'tool', 'ventilation', 'soap bottle', 'bathrobe', 'pictures board', 'cap', 'woofer', 'tshirt', 'rolling bag', 'shoe box', 'luggage bag', 'file storage', 'cat bed', 'stack of paper', 'surfing board', 'electric kettle', 'rolling stand', 'cover', 'main switchboard', 'pressure cooker', 'stepladder', 'countertop', 'flip flops', 'short table', 'sit-up pillow', 'duffel bag', 'shower seating', 'washbasin', 'teddy bear', 'stair', 'plate rack', 'ornament', 'jerrycan', 'filter jug', 't shirt', 'cooking pot', 'platform trolley', 'blinds rod', 'hand shower', 'power socket unit', 'sheep doll', 'laptop bag', 'game console', 'bottles case', 'lid', 'dumbbell case', 'rolled blanket', 'paper stapler', 'kitchen pot', 'charcoal bag', 'laundry hamper', 'rolled poster', 'bath towel', 'apron', 'dustpan', 'trash bag', 'document tray', 'camera', 'mirror cabinet', 'dish drying rack', 'gas tank', 'cable roller', 'case', 'ring light', 'hair dryer', 'gym plate', 'hand towel', 'sill', 'sidetable', 'vice', 'bench stool', 'billboard', 'rolling cabinet', 'shower sink', 'cloth piece', 'oscilloscope', 'magazine rack', 'wash basin', 'cable panel', 'photo frame', 'tv receiver', 'stand', 'milk jug', 'wooden board', 'bladeless fan', 'door frame', 'wall paper', 'scale', 'purse', 'electronic device', 'sofa cushion', 'sponge', 'dish washer', 'crate trolley', 'kitchen hood', 'laundry vent', 'medical stool', 'exhaustive fan', 'portable ladder', 'chemical container', 'toilet paper rolls', 'rag', 'blender', 'window pane', 'dog bowl', 'shopping basket', 'piano stool', 'electric box', 'wall calendar', 'paper holder', 'chemical bottle', 'sandals', 'foreman grill', 'guitar case', 'heater tube', 'running shoes', 'shower tap', 'cloth hanger', 'microphone', 'cabinet frame', 'decorative object', 'light fixture', 'ceiling lamp bar', 'paperbag', 'chemical barrel', 'wicker basket', 'exit sign', 'bottles rack', 'water jug', 'bottle carrier', 'laboratory pellet press', 'mini oven', 'shower arm', 'paper tube', 'suitcase stand', 'table fan', 'shelve', 'full-length mirror', 'wood piece', 'can', 'suit bag', 'water bubbler', 'first aid kit', 'kitchen robot', 'toilet flush button', 'pillow toy', 'plush doll', 'styrofoam box', 'document organizer', 'pet carrier', 'folding table', 'gloves', 'pitcher', 'cable spool', 'rolled cable', 'folding umbrella', 'robot vacuum cleaner', 'tower ventilator', 'brush', 'planter', 'baseball cap', 'gas cylinder', 'stereo', 'baby stroller', 'water bucket', 'rucksack', 'shower door frame', 'drone', 'kitchen cloth', 'hand soap dispenser', 'pegboard', 'alarm', 'emergency light', 'sign board', 'weight plate', 'rolled projection screen', 'laptop table', 'hole puncher', 'mixer', 'piano chair', 'paper bin', 'wooden stick', 'fireplace', 'rolled backdrop', 'mousepad', 'long pillow', 'bananas', 'column', 'cd player', 'eraser', 'laptop sleeve', 'hairdryer', 'seat cushion', 'plant pot mat', 'wastebin', 'wood panel', 'detergent bottle', 'safe box', 'pouch', 'blind rod', 'mop basin', 'plug', 'document holder', 'railing', 'plastic drum', 'cat tree', 'mirror light', 'kettlebell', 'chart', 'dust pan', 'sandal', 'first aid cabinet', 'bracket', 'wire', 'scooter', 'racing wheel', 'wine rack', 'belt', 'tissue dispenser stand', 'towel paper dispenser', 'air heater', 'plushie', 'knife set', 'iron', 'intercom', 'kitchen ceiling', 'package', 'toilet paper dispensor', 'sneakers', 'umbrella stand', 'egg carton', 'organizer', 'stick', 'shampoo bottle', 'cone', 'file tray', 'wooden plan', 'cooking pan', 'brief', 'paper towel package', 'glove dispenser', 'dispenser bottle', 'kitchen drawer', 'remote control', 'powerstrip', 'emergency shower', 'scanner', 'towel holder', 'vr headset', 'watering pot', 'soda machine', 'pole stand', 'roomba', 'rolling mat', 'fluorescent lamp', 'flower', 'pc tower', 'metal mount', 'oven gloves', 'soap', 'flush tank', 'notepad', 'pull up bar', 'loafers', 'water meter cover', 'plastic tray', 'webcam', 'barbell', 'tea pot', 'wall hanger', 'cabinet base panel', 'laptop case', 'paper towel holder', 'skeleton', 'socket extender', 'extension chord reel', 'wooden crate', 'guillotine paper cutter', 'sewing machine', 'model car', 'bed cover', 'storage', 'freezer', 'piano book', 'wifi router', 'overhead shower', 'chrismas tree', 'drill', 'clothes drying stand', 'dumbell', 'cabel', 'badminton racket', 'cool box', 'thermostat', 'dartboard', 'switchboard cover', 'candle', 'electric stove', 'paper ram', 'insulated can', 'prosthetic leg', 'desk power strip', 'closet rail', 'water pitcher', 'plate weights', 'extension cord', 'tea box', 'tissue', 'sauce pan', 'toothbrush', 'frying pan', 'package of paper', 'microphone stand', 'socket box', 'leather mattress', 'plastic can', 'garbage bin cover', 'plush toy', 'electric guitar', 'weight scale', 'fruit', 'tape dispenser', 'pallet', 'emergency kit', 'bathroom mirror', 'door lamp', 'power extension', 'electric circuit board', 'oven panel', 'electrical pipe', 'shade', 'photoframe', 'file stack', 'pizza box', 'tennis racket', 'wall hook', 'recycle bag', 'recessed shower shelve', 'wall mounted telephone', 'facsimile', 'kitchen roll', 'totebag', 'floor cleaner', 'body weight scale', 'stuffed animal door insulator', 'neck pillow', 'basketball', 'cable wheel', 'door vent', 'foot massager', 'pumper', 'hanging light fixture', 'interphone', 'paper towel roll', 'control unit', 'folder oragnizer', 'hanging frame', 'paper stack', 'door handle', 'mini shelf', 'beverage carton', 'wok pan', 'bedside counter', 'bar', 'pot lid', 'radio', 'paper box', 'stabilizer', 'headphone case', 'folded cardboard box', 'coaster', 'pen tray', 'marker', 'shower mat', 'rod', 'device', 'electric pot', 'mixer machine', 'barstool', 'bread toaster', 'electric mixer', 'wall cord cover', 'camera bag', 'charger', 'knife', 'metal frame', 'folded bag', 'conduit pipe', 'french press', 'cabinet side panel', 'cosmetic bag', 'surveillance camera', 'bathroom holder', 'footrest', 'glasses case', 'handbag', 'cooling pad', 'flip flop', 'game controller', 'packet of toilet paper', 'cable duct', 'toilet paper roll', 'monitor support', 'fire alarm', 'kitchen stove', 'key hanger', 'tub', 'network socket', 'ceilng light', 'christmas ornament', 'pepper mill', 'wall outlet', 'light cover', 'caution board', 'heels', 'package bag', 'blinds rail', 'candle holder', 'electric toothbrush', 'tool case', 'toilet flush', 'wooden brush', 'hand washing soap', 'hygiene product', 'water tap', 'cosmetic pouch', 'mount', 'knife holder', 'dustpan and brush', 'circular tray', 'shoe case', 'ar tag', 'flipflop', 'sculpture', 'recycle bin', 'kitchen utensil', 'multiplug', 'beaker stand', 'chessboard', 'pen', 'toothpaste', 'tv remote', 'floor wiper', 'wire hider', 'water meter', 'magazine holder', 'mailbox', 'paper file', 'wall coat hanger', 'utensil holder', 'detergent', 'safe', 'packet', 'dvd', 'dvd player', 'tupperware', 'electrical board', 'radiator pipe', 'plat', 'decorative mirror', 'telephone stand', 'water boiler', 'cutboard', 'hanging deer skull', 'file orginizer', 'satchel', 'head model', 'parcel', 'dish soap bottle', 'glass plate', 'pencil case', 'cleaning mop', 'mixer glass', 'joystick', 'vacuum flask', 'bag of oranges', 'pencil holder', 'pamphlet', 'rope', 'cd', 'knife stand', 'pencil cup', 'binding machine', 'grill pan', 'tape', 'cabinet top panel', 'cutti̇ng board', 'salad spinner', 'water filter jug', 'mop cloth', 'shower valve', 'laptop cover', 'running shoe', 'alligator clips', 'insulated coffee mug', 'pencil stand', 'action figure', 'desk light', 'midi controller', 'bathroom slippers', 'elephant decoration piece', 'switchboard', 'remote', 'arch folder', 'power cord', 'hot bag', 'pencils cup', 'knife block', 'thermos', 'power switch', 'toothbrush holder', 'cleaning liquid', 'power board', 'pull-up bar', 'sandwich maker', 'statue', 'brief case', 'airdyer', 'bike helmet', 'mirror lamp', 'tennis rackets', 'paint jar', 'citrus juicer', 'scissors', 'hanging hook', 'napkin', 'laundry detergent', 'disinfectant dispenser', 'cleaning brush', 'food', 'pan set', 'kitchen appliance', 'mop pad', 'dish soap', 'plunger', 'paper package', 'plastic mat', 'calculator', 'voltage stabilizer', 'whiteboard marker', 'wall coat rack', 'milk carton', 'cosmetic bottle', 'shower handle', 'tissue paper', 'shorts', 'game console controller', 'remote controller', 'teapot', 'drain', 'juice box', 'phone stand', 'soap container', 'flush', 'rubber water bag', 'shower gel', 'snack bag', 'watering bucket', 'first aid box', 'banana', 'paper notebook', 'hammer', 'kitchen glove', 'handle', 'personal hygiene product', 'duster', 'robot vaccuum cleaner', 'chain', 'soap dish', 'cereal box', 'door window', 'elephant toy', 'frisbee', 'cream tube', 'folder holder', 'internet socket', 'moka pot', 'stereo box', 'face mask', 'bunny chocolate', 'magazine collector', 'shower holder', 'toilet brush', 'oven glove', 'shower drain', 'penholder', 'switch board', 'coffee pot', 'marker eraser', 'squeegee', 'spray', 'game cd box', 'table clock', 'apple', 'spool', 'portable speaker', 'guitar pedal', 'bread', 'glass jar', 'paper puncher', 'dishwashing sponge', 'shower hose', 'surface cleaning liquid', 'wallet', 'circuit box', 'toiletry', 'hair brush', 'headphone bag', 'metal saw', 'power adapter', 'wlan router', 'plant pot coaster', 'oven mitt', 'hanging light', 'kitchen object', 'machine button', 'vanity light', 'flush plate', 'leaf fan', 'punching machine', 'window handle', 'wine bottle', 'tennis cap', 'chips can', 'food pot', 'hand washing soap dispenser', 'wall light', 'detergent bag', 'glasses cover', 'hangbag', 'bottle spray', 'kitchen tap', 'knob', 'figurine', 'hand vacuum', 'potted plant', 'marker storage', 'mixing bowl', 'power supply', 'intercom screen', 'coffee mug', 'drilling machine', 'electric socket', 'hand shower handle', 'hole punch', 'silicon gun', 'document holder', 'wooden chest', 'cheese', 'headphones case', 'measuring spoon', 'monitor light', 'ear muffs', 'night lamp', 'wall handle', 'intercom device', 'lanyard', 'rugby ball', 'shower loofah', 'cable socket', 'cleaning cloth', 'dish washer soap', 'power brick', 'coffee', 'monitor base', 'mushroom lamp', 'trivet', 'surface cleaner', 'toiletry bottle', 'paper note', 'strainer', 'colander', 'kitchen brush', 'multi socket', 'saucer', 'scissor', 'stamp', 'emergency button', 'toilet brush holder', 'coffee jar', 'deodorant', 'shower rod', 'wet tissue', 'tissue paper roll', 'funnel', 'oragnizer', 'toilet plunger', 'tumbler', 'door knob', 'coarser', 'postit note', 'punching tool', 'beverage can', 'dish', 'star', 'flask', 'in-table power socket', 'laptop charger', 'wall intercom', 'flour bag', 'blowtorch', 'cleaner', 'takeout box', 'hand soap', 'hard drive', 'monitor holder', 'soldering iron', 'control switch', 'drainage', 'box lid', 'playstation controller', 'shaving foam', 'guitar stand', 'toilet cleaner', 'co detector', 'covered power socket', 'spray can', 'tooth brush', 'phone tripod', 'bluetooth speaker', 'dish washing liquid', 'powerbank', 'dried plant', 'shampoo', 'sticky note', 'toilet seat brush', 'dishwashing soap', 'lan port', 'scrubber', 'cord', 'ashtray', 'handle bar', 'dongle', 'liquid soap', 'mobile phone', 'bunny decoration', 'palm rest', 'stream deck', 'blinds chain', 'dishwashing liquid', 'hammer holder', 'hand washing liquid', 'shower wiper', 'duct tape', 'valve', 'glove', 'stationery', 'ceiling speaker', 'holder', 'power plug', 'wall speaker', 'card', 'electrical tape', 'pot cover', 'razor', 'barcode scanner', 'lotion', 'grab rail', 'hairbrush', 'lint roller', 'post-it', 'dusting cloth', 'glue bottle', 'handwash', 'network outlet', 'card reader', 'kitchen light', 'pc charger', 'product dispenser bottle', 'hair dryer holder', 'screwdriver', 'shower squeegee', 'juice tetrapack', 'pliers', 'sunblock', 'bed sheet', 'hook', 'headphone holder', 'light bulb', 'ladle', 'bar soap', 'button', 'dish brush', 'toy car', 'note', 'post it', 'sponge cloth', 'wood stick', 'paper weight', 'product tube', 'cell phone', 'smartphone', 'comb', 'jump rope', 'lan', 'wireless charger', 'door stopper', 'cigarette packet', 'pencil', 'soap holder', 'electrical plug', 'toilet holder', 'mask', 'pocket calculator', 'cabbage', 'spoon', 'doorknob', 'fan switch', 'rubber duck', 'backdrop hook', 'table tennis racket', 'receipt', 'soap bar', 'tape roll', 'tooth paste', 'electrical adapter', 'probe', 'highlighter', 'correction fluid', 'dispenser', 'pi̇cture', 'door hinge', 'whiteboard marker holder', 'botttle', 'power outlet', 'towel hanger', 'whiteboard duster', 'magnet', 'sticker', 'nose spray', 'vertical blind control', 'razor blade', 'post it note', 'wireless headphones', 'cabinet top', 'cream bottle', 'datashow socket', 'earbuds', 'cabinet door', 'chair cushion', 'cosmetic tube'] + +SCANNETPP_IDS = [3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 115, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 228, 229, 230, 231, 232, 233, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 249, 250, 251, 252, 253, 255, 256, 257, 258, 259, 260, 261, 262, 263, 264, 265, 266, 267, 268, 269, 270, 271, 272, 273, 274, 275, 276, 277, 278, 279, 280, 281, 282, 283, 284, 285, 286, 287, 288, 289, 290, 291, 292, 293, 294, 295, 296, 297, 298, 299, 300, 301, 302, 303, 304, 305, 306, 307, 308, 309, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, 325, 326, 327, 328, 329, 330, 331, 332, 333, 334, 335, 336, 337, 338, 339, 340, 341, 342, 343, 344, 345, 346, 348, 349, 350, 351, 352, 353, 354, 355, 356, 357, 358, 359, 360, 361, 362, 363, 364, 365, 366, 367, 368, 369, 370, 371, 372, 373, 374, 375, 377, 378, 379, 380, 381, 382, 383, 384, 385, 386, 387, 388, 389, 390, 391, 392, 393, 394, 395, 396, 398, 399, 400, 401, 402, 403, 404, 405, 406, 407, 408, 409, 410, 411, 412, 413, 414, 415, 416, 417, 418, 419, 420, 421, 422, 423, 424, 425, 426, 427, 428, 429, 430, 431, 432, 433, 435, 436, 437, 438, 439, 440, 441, 442, 443, 444, 445, 446, 447, 448, 449, 450, 451, 453, 454, 455, 456, 457, 458, 459, 460, 461, 462, 463, 464, 465, 466, 467, 468, 469, 470, 471, 472, 473, 474, 475, 476, 477, 478, 479, 480, 481, 482, 483, 484, 485, 486, 487, 488, 489, 490, 491, 492, 493, 495, 496, 499, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511, 512, 513, 514, 515, 517, 518, 519, 520, 521, 522, 523, 524, 525, 526, 527, 528, 529, 531, 532, 533, 534, 535, 536, 537, 538, 539, 540, 541, 542, 543, 544, 545, 546, 547, 548, 549, 550, 551, 552, 553, 554, 556, 557, 558, 559, 560, 561, 562, 563, 564, 565, 566, 567, 568, 569, 570, 571, 572, 575, 576, 577, 578, 580, 581, 583, 584, 585, 586, 588, 589, 591, 592, 593, 594, 595, 596, 597, 598, 602, 603, 604, 605, 606, 607, 608, 609, 611, 612, 613, 614, 615, 616, 617, 618, 620, 621, 622, 623, 624, 625, 626, 627, 628, 629, 630, 631, 632, 633, 634, 635, 636, 637, 638, 639, 640, 641, 642, 643, 644, 646, 647, 648, 649, 650, 651, 653, 654, 655, 656, 657, 658, 659, 661, 662, 663, 664, 665, 666, 667, 668, 669, 670, 671, 673, 674, 675, 676, 677, 678, 679, 680, 681, 682, 683, 684, 685, 686, 687, 690, 691, 692, 693, 694, 695, 696, 697, 698, 699, 700, 701, 702, 703, 705, 706, 709, 710, 711, 712, 713, 714, 716, 717, 718, 719, 720, 721, 722, 723, 724, 726, 727, 728, 729, 730, 731, 732, 733, 734, 735, 736, 737, 738, 739, 740, 741, 742, 743, 744, 745, 746, 747, 748, 749, 750, 751, 752, 753, 754, 755, 757, 758, 759, 760, 761, 762, 763, 764, 765, 766, 767, 768, 769, 770, 771, 772, 773, 774, 775, 776, 777, 778, 779, 780, 781, 782, 783, 784, 785, 786, 787, 788, 789, 790, 791, 792, 793, 794, 795, 796, 797, 798, 799, 800, 801, 802, 803, 804, 805, 806, 807, 808, 809, 810, 811, 812, 813, 814, 815, 816, 817, 818, 819, 820, 821, 822, 823, 824, 825, 826, 827, 828, 829, 830, 831, 832, 833, 834, 836, 837, 838, 839, 840, 841, 842, 843, 844, 845, 846, 847, 848, 849, 850, 851, 852, 853, 855, 856, 857, 858, 859, 860, 861, 862, 863, 864, 865, 866, 867, 868, 869, 870, 871, 873, 874, 875, 876, 877, 878, 879, 880, 881, 882, 883, 885, 886, 887, 888, 889, 890, 891, 892, 893, 894, 895, 896, 897, 898, 899, 900, 901, 902, 903, 904, 906, 907, 908, 909, 910, 911, 912, 913, 914, 915, 916, 917, 919, 920, 921, 922, 923, 924, 925, 926, 928, 929, 932, 933, 934, 935, 936, 937, 938, 939, 940, 941, 943, 944, 946, 947, 948, 949, 950, 951, 953, 954, 955, 956, 957, 958, 959, 960, 961, 962, 963, 964, 965, 966, 967, 968, 969, 970, 971, 972, 973, 974, 977, 979, 980, 981, 982, 983, 984, 985, 986, 987, 988, 989, 990, 991, 992, 993, 994, 995, 996, 997, 998, 999, 1000, 1001, 1002, 1003, 1004, 1005, 1006, 1007, 1008, 1010, 1011, 1012, 1013, 1014, 1015, 1016, 1017, 1018, 1019, 1020, 1021, 1022, 1023, 1024, 1025, 1026, 1028, 1029, 1030, 1031, 1032, 1033, 1034, 1035, 1036, 1037, 1038, 1039, 1040, 1041, 1042, 1043, 1044, 1045, 1046, 1047, 1048, 1049, 1050, 1051, 1052, 1053, 1054, 1055, 1056, 1057, 1058, 1059, 1060, 1061, 1062, 1063, 1064, 1065, 1066, 1067, 1068, 1069, 1070, 1071, 1072, 1073, 1074, 1076, 1077, 1078, 1080, 1081, 1082, 1083, 1085, 1086, 1087, 1089, 1091, 1092, 1093, 1094, 1095, 1096, 1097, 1098, 1099, 1100, 1101, 1102, 1103, 1104, 1105, 1107, 1108, 1109, 1110, 1111, 1113, 1114, 1115, 1116, 1117, 1118, 1119, 1120, 1121, 1122, 1123, 1125, 1126, 1127, 1128, 1130, 1131, 1132, 1133, 1134, 1135, 1136, 1137, 1138, 1139, 1140, 1141, 1142, 1143, 1144, 1145, 1146, 1147, 1148, 1149, 1150, 1151, 1152, 1153, 1154, 1155, 1156, 1157, 1158, 1159, 1160, 1161, 1162, 1163, 1164, 1165, 1166, 1167, 1168, 1169, 1170, 1171, 1172, 1173, 1174, 1175, 1176, 1177, 1178, 1179, 1180, 1181, 1182, 1183, 1184, 1185, 1186, 1187, 1188, 1189, 1190, 1191, 1192, 1193, 1194, 1195, 1196, 1197, 1198, 1199, 1200, 1201, 1202, 1203, 1205, 1206, 1207, 1208, 1209, 1210, 1211, 1212, 1214, 1215, 1216, 1217, 1218, 1219, 1220, 1221, 1222, 1223, 1224, 1225, 1226, 1227, 1228, 1229, 1230, 1231, 1232, 1233, 1234, 1235, 1236, 1237, 1238, 1239, 1240, 1242, 1243, 1244, 1245, 1246, 1247, 1248, 1249, 1250, 1251, 1252, 1253, 1254, 1255, 1256, 1257, 1258, 1259, 1260, 1261, 1262, 1263, 1264, 1265, 1266, 1267, 1268, 1269, 1270, 1271, 1272, 1273, 1274, 1275, 1276, 1277, 1279, 1280, 1281, 1283, 1284, 1285, 1286, 1287, 1288, 1289, 1290, 1291, 1292, 1293, 1294, 1295, 1296, 1297, 1298, 1299, 1300, 1301, 1302, 1303, 1304, 1305, 1306, 1307, 1308, 1309, 1310, 1311, 1312, 1313, 1314, 1315, 1316, 1317, 1318, 1319, 1320, 1321, 1322, 1323, 1324, 1325, 1326, 1327, 1328, 1329, 1330, 1331, 1332, 1333, 1334, 1335, 1336, 1337, 1338, 1339, 1340, 1341, 1342, 1343, 1344, 1345, 1347, 1348, 1349, 1350, 1351, 1352, 1353, 1354, 1355, 1356, 1357, 1358, 1359, 1360, 1361, 1362, 1363, 1364, 1365, 1366, 1367, 1368, 1369, 1370, 1371, 1373, 1374, 1375, 1376, 1377, 1378, 1379, 1380, 1381, 1383, 1384, 1385, 1386, 1387, 1388, 1389, 1390, 1391, 1392, 1393, 1394, 1395, 1396, 1397, 1398, 1399, 1400, 1402, 1403, 1404, 1405, 1406, 1407, 1408, 1409, 1410, 1411, 1413, 1414, 1415, 1417, 1418, 1419, 1420, 1421, 1423, 1425, 1426, 1427, 1428, 1429, 1430, 1431, 1432, 1433, 1435, 1436, 1437, 1438, 1439, 1440, 1441, 1442, 1443, 1444, 1445, 1446, 1447, 1449, 1450, 1451, 1452, 1453, 1454, 1455, 1456, 1457, 1458, 1459, 1460, 1461, 1463, 1464, 1465, 1466, 1467, 1468, 1469, 1470, 1471, 1472, 1473, 1474, 1476, 1477, 1478, 1479, 1480, 1481, 1482, 1483, 1484, 1485, 1486, 1487, 1488, 1489, 1490, 1491, 1492, 1493, 1494, 1495, 1496, 1497, 1498, 1499, 1500, 1501, 1502, 1503, 1504, 1505, 1506, 1508, 1509, 1510, 1511, 1512, 1513, 1514, 1515, 1516, 1517, 1518, 1519, 1520, 1521, 1522, 1525, 1526, 1527, 1528, 1529, 1530, 1531, 1532, 1533, 1535, 1536, 1537, 1539, 1540, 1541, 1542, 1543, 1544, 1545, 1546, 1547, 1548, 1549, 1550, 1551, 1552, 1553, 1554, 1555, 1556, 1557, 1558, 1559, 1560, 1561, 1562, 1563, 1565, 1566, 1567, 1568, 1570, 1571, 1572, 1573, 1574, 1576, 1578, 1579, 1580, 1581, 1582, 1583, 1584, 1585, 1586, 1589, 1590, 1591, 1593, 1594, 1595, 1596, 1597, 1598, 1599, 1600, 1601, 1603, 1604, 1605, 1606, 1607, 1608, 1609, 1610, 1611, 1612, 1613, 1614, 1615, 1616, 1617, 1618, 1619, 1620, 1621, 1622, 1623, 1624, 1625, 1627, 1628, 1629, 1630, 1631, 1632, 1633, 1634, 1635, 1636, 1637, 1638, 1639, 1641, 1642, 1643, 1644, 1645, 1646, 1648, 1649, 1650, 1651, 1652, 1653, 1654, 1655, 1656, 1657, 1658] + +SCANNETPP84_LABELS = ['table', 'door', 'ceiling lamp', 'cabinet', 'blinds', 'curtain', 'chair', 'storage cabinet', 'office chair', 'bookshelf', 'whiteboard', 'window', 'box', + 'monitor', 'shelf', 'heater', 'kitchen cabinet', 'sofa', 'bed', 'trash can', 'book', 'plant', 'blanket', 'tv', 'computer tower', 'refrigerator', 'jacket', + 'sink', 'bag', 'picture', 'pillow', 'towel', 'suitcase', 'backpack', 'crate', 'keyboard', 'rack', 'toilet', 'printer', 'poster', 'painting', 'microwave', 'shoes', + 'socket', 'bottle', 'bucket', 'cushion', 'basket', 'shoe rack', 'telephone', 'file folder', 'laptop', 'plant pot', 'exhaust fan', 'cup', 'coat hanger', 'light switch', + 'speaker', 'table lamp', 'kettle', 'smoke detector', 'container', 'power strip', 'slippers', 'paper bag', 'mouse', 'cutting board', 'toilet paper', 'paper towel', + 'pot', 'clock', 'pan', 'tap', 'jar', 'soap dispenser', 'binder', 'bowl', 'tissue box', 'whiteboard eraser', 'toilet brush', 'spray bottle', 'headphones', 'stapler', 'marker'] + +SCANNETPP84_IDS = [4, 3, 6, 5, 9, 7, 8, 10, 12, 11, 14, 13, 23, 17, 18, 24, 25, 27, 28, 47, 88, 35, 36, 42, 45, 58, 49, 54, 56, 59, 60, 63, 67, 68, 102, 71, 72, 74, 81, 83, 90, 96, 122, 416, 106, 111, 117, 126, 129, 132, 155, 166, 173, 188, 300, 199, 204, 214, 219, 253, 299, 265, 273, 352, 295, 296, 301, 305, 312, 342, 358, 364, 368, 387, 395, 396, 403, 405, 414, 443, 469, 515, 744, 1157] + + +SCANNET20_LABELS = ['toilet', 'bed', 'chair', 'sofa', 'dresser', 'table', 'cabinet', 'bookshelf', 'pillow', 'sink', 'bathtub', 'refrigerator', 'desk', 'nightstand', 'counter', 'door', 'curtain', 'box', 'lamp', 'bag'] + +SCANNET20_IDS = list(range(20)) + +ARKIT_LABELS = ['cabinet', 'refrigerator', 'shelf', 'stove', 'bed', + 'sink', 'washer', 'toilet', 'bathtub', 'oven', + 'dishwasher', 'fireplace', 'stool', 'chair', 'table', + 'tv_monitor', 'sofa'] + +ARKIT_IDS = list(range(len(ARKIT_LABELS))) \ No newline at end of file diff --git a/MaskClustering/evaluation/evaluate.py b/MaskClustering/evaluation/evaluate.py new file mode 100755 index 0000000000000000000000000000000000000000..4e3ba15bc83abc8dc5e29c8d4476c37585dc4bca --- /dev/null +++ b/MaskClustering/evaluation/evaluate.py @@ -0,0 +1,420 @@ +import os, sys, argparse +from copy import deepcopy +import numpy as np +import torch +from evaluation.utils_3d import get_instances + +parser = argparse.ArgumentParser() +parser.add_argument('--pred_path', required=True, help='path to directory of predicted .txt files') +parser.add_argument('--gt_path', required=True, help='path to directory of ground truth .txt files') +parser.add_argument('--dataset', required=True, help='type of dataset, e.g. matterport3d, scannet, etc.') +parser.add_argument('--output_file', default='', help='path to output file') +parser.add_argument('--no_class', action='store_true', help='class agnostic evaluation') +opt = parser.parse_args() + +# ---------- Label info ---------- # +from evaluation.constants import MATTERPORT_LABELS, MATTERPORT_IDS, SCANNET_LABELS, SCANNET_IDS, SCANNETPP_LABELS, SCANNETPP_IDS + +if opt.dataset == 'matterport3d': + CLASS_LABELS = MATTERPORT_LABELS + VALID_CLASS_IDS = MATTERPORT_IDS +elif opt.dataset == 'scannet': + CLASS_LABELS = SCANNET_LABELS + VALID_CLASS_IDS = SCANNET_IDS +elif opt.dataset == 'scannetpp': + CLASS_LABELS = SCANNETPP_LABELS + VALID_CLASS_IDS = SCANNETPP_IDS + + +if opt.output_file == '': + opt.output_file = os.path.join(f'data/evaluation/{opt.dataset}', opt.pred_path.split('/')[-1] + '.txt') + os.makedirs(os.path.dirname(opt.output_file), exist_ok=True) +if opt.no_class: + if 'class_agnostic' not in opt.output_file: + opt.output_file = opt.output_file.replace('.txt', '_class_agnostic.txt') + +ID_TO_LABEL = {} +LABEL_TO_ID = {} +for i in range(len(VALID_CLASS_IDS)): + LABEL_TO_ID[CLASS_LABELS[i]] = VALID_CLASS_IDS[i] + ID_TO_LABEL[VALID_CLASS_IDS[i]] = CLASS_LABELS[i] + +# ---------- Evaluation params ---------- # +# overlaps for evaluation +opt.overlaps = np.append(np.arange(0.5,0.95,0.05), 0.25) +# minimum region size for evaluation [verts] +opt.min_region_sizes = np.array( [ 100 ] ) +# distance thresholds [m] +opt.distance_threshes = np.array( [ float('inf') ] ) +# distance confidences +opt.distance_confs = np.array( [ -float('inf') ] ) + + +def evaluate_matches(matches): + overlaps = opt.overlaps + min_region_sizes = [ opt.min_region_sizes[0] ] + dist_threshes = [ opt.distance_threshes[0] ] + dist_confs = [ opt.distance_confs[0] ] + + # results: class x overlap + ap = np.zeros( (len(dist_threshes) , len(CLASS_LABELS) , len(overlaps)) , float ) + for di, (min_region_size, distance_thresh, distance_conf) in enumerate(zip(min_region_sizes, dist_threshes, dist_confs)): + for oi, overlap_th in enumerate(overlaps): + pred_visited = {} + for m in matches: + for p in matches[m]['pred']: + for label_name in CLASS_LABELS: + for p in matches[m]['pred'][label_name]: + if 'filename' in p: + pred_visited[p['filename']] = False + for li, label_name in enumerate(CLASS_LABELS): + y_true = np.empty(0) + y_score = np.empty(0) + hard_false_negatives = 0 + has_gt = False + has_pred = False + for m in matches: + pred_instances = matches[m]['pred'][label_name] + gt_instances = matches[m]['gt'][label_name] + # filter groups in ground truth + gt_instances = [ gt for gt in gt_instances if gt['instance_id']>=1000 and gt['vert_count']>=min_region_size and gt['med_dist']<=distance_thresh and gt['dist_conf']>=distance_conf ] + if gt_instances: + has_gt = True + if pred_instances: + has_pred = True + + cur_true = np.ones ( len(gt_instances) ) + cur_score = np.ones ( len(gt_instances) ) * (-float("inf")) + cur_match = np.zeros( len(gt_instances) , dtype=bool ) + # collect matches + for (gti,gt) in enumerate(gt_instances): + found_match = False + num_pred = len(gt['matched_pred']) + for pred in gt['matched_pred']: + # greedy assignments + if pred_visited[pred['filename']]: + continue + overlap = float(pred['intersection']) / (gt['vert_count']+pred['vert_count']-pred['intersection']) + if overlap > overlap_th: + confidence = pred['confidence'] + # if already have a prediction for this gt, + # the prediction with the lower score is automatically a false positive + if cur_match[gti]: + max_score = max( cur_score[gti] , confidence ) + min_score = min( cur_score[gti] , confidence ) + cur_score[gti] = max_score + # append false positive + cur_true = np.append(cur_true,0) + cur_score = np.append(cur_score,min_score) + cur_match = np.append(cur_match,True) + # otherwise set score + else: + found_match = True + cur_match[gti] = True + cur_score[gti] = confidence + pred_visited[pred['filename']] = True + + + if not found_match: + hard_false_negatives += 1 + # remove non-matched ground truth instances + cur_true = cur_true [ cur_match==True ] + cur_score = cur_score[ cur_match==True ] + + # collect non-matched predictions as false positive + for pred in pred_instances: + found_gt = False + for gt in pred['matched_gt']: + overlap = float(gt['intersection']) / (gt['vert_count']+pred['vert_count']-gt['intersection']) + if overlap > overlap_th: + found_gt = True + break + if not found_gt: + num_ignore = pred['void_intersection'] + for gt in pred['matched_gt']: + # group? + if gt['instance_id'] < 1000: + num_ignore += gt['intersection'] + # small ground truth instances + if gt['vert_count'] < min_region_size or gt['med_dist']>distance_thresh or gt['dist_conf'] 0, then the prediction is considered a match + ''' + pred_info = read_pridiction_npz(os.path.join(pred_file)) + gt_ids = np.loadtxt(gt_file) + + if opt.no_class: + gt_ids = gt_ids % 1000 + VALID_CLASS_IDS[0] * 1000 + + # get gt instances + gt_instances = get_instances(gt_ids, VALID_CLASS_IDS, CLASS_LABELS, ID_TO_LABEL) + # associate + gt2pred = deepcopy(gt_instances) + for label in gt2pred: + for gt in gt2pred[label]: + gt['matched_pred'] = [] + pred2gt = {} + for label in CLASS_LABELS: + pred2gt[label] = [] + num_pred_instances = 0 + # mask of void labels in the groundtruth + bool_void = np.logical_not(np.in1d(gt_ids//1000, VALID_CLASS_IDS)) + + gt_tensor_dict = get_gt_tensor(gt_ids, gt_instances) + + # go thru all prediction masks + for pred_mask_file in (pred_info): + if opt.no_class: + label_id = VALID_CLASS_IDS[0] + else: + label_id = int(pred_info[pred_mask_file]['label_id']) + conf = pred_info[pred_mask_file]['conf'] + if not label_id in ID_TO_LABEL: + continue + label_name = ID_TO_LABEL[label_id] + # read the mask + pred_mask = pred_info[pred_mask_file]['mask'] + + if len(pred_mask) != len(gt_ids): + print('wrong number of lines in ' + pred_mask_file + '(%d) vs #mesh vertices (%d), please double check and/or re-download the mesh' % (len(pred_mask), len(gt_ids))) + raise NotImplementedError + + # convert to binary + pred_mask = np.not_equal(pred_mask, 0) + num = np.count_nonzero(pred_mask) + if num < opt.min_region_sizes[0]: + continue # skip if empty + + pred_instance = {} + pred_instance['filename'] = pred_mask_file + pred_instance['pred_id'] = num_pred_instances + pred_instance['label_id'] = label_id + pred_instance['vert_count'] = num + pred_instance['confidence'] = conf + pred_instance['void_intersection'] = np.count_nonzero(np.logical_and(bool_void, pred_mask)) + + # matched gt instances + matched_gt = [] + gt_tensor = gt_tensor_dict[label_name] + intersection = torch.sum(gt_tensor & torch.from_numpy(pred_mask).cuda().reshape(-1, 1), dim=0) + intersect_ids = torch.nonzero(intersection).cpu().numpy().reshape(-1) + for gt_id in intersect_ids: + gt_copy = gt_instances[label_name][gt_id].copy() + pred_copy = pred_instance.copy() + intersection_num = intersection[gt_id].item() + gt_copy['intersection'] = intersection_num + pred_copy['intersection'] = intersection_num + matched_gt.append(gt_copy) + gt2pred[label_name][gt_id]['matched_pred'].append(pred_copy) + + pred_instance['matched_gt'] = matched_gt + num_pred_instances += 1 + pred2gt[label_name].append(pred_instance) + + return gt2pred, pred2gt + +def print_results(avgs): + sep = "" + col1 = ":" + lineLen = 64 + + print ("") + print ("#"*lineLen) + line = "" + line += "{:<15}".format("what" ) + sep + col1 + line += "{:>15}".format("AP" ) + sep + line += "{:>15}".format("AP_50%" ) + sep + line += "{:>15}".format("AP_25%" ) + sep + print (line) + print ("#"*lineLen) + + for (li,label_name) in enumerate(CLASS_LABELS): + ap_avg = avgs["classes"][label_name]["ap"] + if np.isnan(ap_avg): + continue + ap_50o = avgs["classes"][label_name]["ap50%"] + ap_25o = avgs["classes"][label_name]["ap25%"] + line = "{:<15}".format(label_name) + sep + col1 + line += sep + "{:>15.3f}".format(ap_avg ) + sep + line += sep + "{:>15.3f}".format(ap_50o ) + sep + line += sep + "{:>15.3f}".format(ap_25o ) + sep + print (line) + + all_ap_avg = avgs["all_ap"] + all_ap_50o = avgs["all_ap_50%"] + all_ap_25o = avgs["all_ap_25%"] + + print ("-"*lineLen) + line = "{:<15}".format("average") + sep + col1 + line += "{:>15.3f}".format(all_ap_avg) + sep + line += "{:>15.3f}".format(all_ap_50o) + sep + line += "{:>15.3f}".format(all_ap_25o) + sep + print (line) + print ("") + +def write_result_file(avgs, filename): + _SPLITTER = ',' + with open(filename, 'w') as f: + f.write(_SPLITTER.join(['class', 'class id', 'ap', 'ap50', 'ap25']) + '\n') + for i in range(len(VALID_CLASS_IDS)): + class_name = CLASS_LABELS[i] + class_id = VALID_CLASS_IDS[i] + ap = avgs["classes"][class_name]["ap"] + ap50 = avgs["classes"][class_name]["ap50%"] + ap25 = avgs["classes"][class_name]["ap25%"] + f.write(_SPLITTER.join([str(x) for x in [class_name, class_id, ap, ap50, ap25]]) + '\n') + f.write(_SPLITTER.join([str(x) for x in [avgs["all_ap"], avgs["all_ap_50%"], avgs["all_ap_25%"]]]) + '\n') + +def evaluate(pred_files, gt_files, pred_path, output_file): + print ('evaluating', len(pred_files), 'scans...') + matches = {} + for i in range(len(pred_files)): + matches_key = os.path.abspath(gt_files[i]) + # assign gt to predictions + gt2pred, pred2gt = assign_instances_for_scan(pred_files[i], gt_files[i]) + matches[matches_key] = {} + matches[matches_key]['gt'] = gt2pred + matches[matches_key]['pred'] = pred2gt + sys.stdout.write("\rscans processed: {}".format(i+1)) + sys.stdout.flush() + ap_scores = evaluate_matches(matches) + avgs = compute_averages(ap_scores) + + # print + print_results(avgs) + write_result_file(avgs, output_file) + +def main(): + print('start evaluating:', opt.pred_path.split('/')[-1]) + pred_files = [f for f in sorted(os.listdir(opt.pred_path)) if f.endswith('.npz') and not f.startswith('semantic_instance_evaluation')] + gt_files = [] + + for i in range(len(pred_files)): + gt_file = os.path.join(opt.gt_path, pred_files[i].replace('.npz', '.txt')) + if not os.path.isfile(gt_file): + print('Result file {} does not match any gt file'.format(pred_files[i])) + raise NotImplementedError + + gt_files.append(gt_file) + pred_files[i] = os.path.join(opt.pred_path, pred_files[i]) + + evaluate(pred_files, gt_files, opt.pred_path, opt.output_file) + print('save results to', opt.output_file) + +if __name__ == '__main__': + main() diff --git a/MaskClustering/evaluation/utils_3d.py b/MaskClustering/evaluation/utils_3d.py new file mode 100644 index 0000000000000000000000000000000000000000..84f729d692870ae6de6f52608a58f9e5cff2b645 --- /dev/null +++ b/MaskClustering/evaluation/utils_3d.py @@ -0,0 +1,66 @@ +import json +import numpy as np + +def load_ids(filename): + ids = open(filename).read().splitlines() + ids = np.array(ids, dtype=np.int64) + return ids + +# ------------ Instance Utils ------------ # + +class Instance(object): + instance_id = 0 + label_id = 0 + vert_count = 0 + med_dist = -1 + dist_conf = 0.0 + + def __init__(self, mesh_vert_instances, instance_id): + if (instance_id == -1): + return + self.instance_id = int(instance_id) + self.label_id = int(self.get_label_id(instance_id)) + self.vert_count = int(self.get_instance_verts(mesh_vert_instances, instance_id)) + + def get_label_id(self, instance_id): + return int(instance_id // 1000) + + def get_instance_verts(self, mesh_vert_instances, instance_id): + return (mesh_vert_instances == instance_id).sum() + + def to_json(self): + return json.dumps(self, default=lambda o: o.__dict__, sort_keys=True, indent=4) + + def to_dict(self): + dict = {} + dict["instance_id"] = self.instance_id + dict["label_id"] = self.label_id + dict["vert_count"] = self.vert_count + dict["med_dist"] = self.med_dist + dict["dist_conf"] = self.dist_conf + return dict + + def from_json(self, data): + self.instance_id = int(data["instance_id"]) + self.label_id = int(data["label_id"]) + self.vert_count = int(data["vert_count"]) + if ("med_dist" in data): + self.med_dist = float(data["med_dist"]) + self.dist_conf = float(data["dist_conf"]) + + def __str__(self): + return "("+str(self.instance_id)+")" + +def get_instances(ids, class_ids, class_labels, id2label): + instances = {} + for label in class_labels: + instances[label] = [] + instance_ids = np.unique(ids) + for id in instance_ids: + if id == 0: + continue + inst = Instance(ids, id) + if inst.label_id in class_ids: + instances[id2label[inst.label_id]].append(inst.to_dict()) + return instances + \ No newline at end of file diff --git a/MaskClustering/infer_single_scene.py b/MaskClustering/infer_single_scene.py new file mode 100644 index 0000000000000000000000000000000000000000..82a83e29e7f1bafdc8ccb080f7e17279354939c2 --- /dev/null +++ b/MaskClustering/infer_single_scene.py @@ -0,0 +1,355 @@ +import os +import argparse +import numpy as np +import time +import shutil +import torch +import urllib.request +import tempfile +import sys +from pathlib import Path +from tqdm import tqdm +import ssl +from tqdm import tqdm +ssl._create_default_https_context = ssl._create_unverified_context + +# Константы ScanNet +BASE_URL = 'http://kaldir.vc.in.tum.de/scannet/' +TOS_URL = BASE_URL + 'ScanNet_TOS.pdf' +FILETYPES = ['.aggregation.json', '.sens', '.txt', '_vh_clean_2.0.010000.segs.json', '_vh_clean_2.ply', '_vh_clean.aggregation.json', '_vh_clean_2.labels.ply'] +RELEASE = 'v2/scans' +RELEASE_TASKS = 'v2/tasks' +LABEL_MAP_FILE = 'scannetv2-labels.combined.tsv' + +# Пути по умолчанию +DEFAULT_CONFIG = "scannet" # Конфигурация по умолчанию +CUDA_ID = 0 # ID используемой GPU + +def parse_args(): + parser = argparse.ArgumentParser(description="MaskClustering на одной сцене") + parser.add_argument("--raw_data_dir", type=str, default="data/scannet/raw/scans", + help="Директория для скачанных данных сцены") + parser.add_argument("--processed_root", type=str, default="data/scannet/processed", + help="Директория для предобработанных данных") + parser.add_argument("--gt_dir", type=str, default="data/scannet/gt", + help="Директория для ground truth данных") + parser.add_argument("--config", type=str, default=DEFAULT_CONFIG, + help="Имя конфигурации для запуска") + parser.add_argument("--cropformer_path", type=str, + default="Mask2Former_hornet_3x_576d0b.pth", + help="Путь к весам CropFormer") + parser.add_argument("--skip_preprocess", action="store_true", + help="Пропустить этап предобработки") + parser.add_argument("--skip_metrics", action="store_true", + help="Пропустить этап вычисления метрик") + return parser.parse_args() + +# Функции для скачивания данных из download-scannet.py +def get_release_scans(release_file): + scan_lines = urllib.request.urlopen(release_file) + scans = [] + for scan_line in scan_lines: + scan_id = scan_line.decode('utf8').rstrip('\n') + scans.append(scan_id) + return scans + +def download_file(url, out_file): + out_dir = os.path.dirname(out_file) + if not os.path.isdir(out_dir): + os.makedirs(out_dir) + if not os.path.isfile(out_file): + print('\t' + url + ' > ' + out_file) + fh, out_file_tmp = tempfile.mkstemp(dir=out_dir) + f = os.fdopen(fh, 'w') + f.close() + try: + urllib.request.urlretrieve(url, out_file_tmp) + os.rename(out_file_tmp, out_file) + except urllib.error.HTTPError as e: + print(f"Ошибка HTTP при скачивании {url}: {e.code} {e.reason}") + if os.path.exists(out_file_tmp): + os.remove(out_file_tmp) + return False + except urllib.error.URLError as e: + print(f"Ошибка URL при скачивании {url}: {e.reason}") + if os.path.exists(out_file_tmp): + os.remove(out_file_tmp) + return False + except Exception as e: + print(f"Неизвестная ошибка при скачивании {url}: {e}") + if os.path.exists(out_file_tmp): + os.remove(out_file_tmp) + return False + else: + print('Файл уже существует: ' + out_file) + return True + +def download_scan(scan_id, out_dir, file_types): + print(f'Скачивание сцены ScanNet {scan_id}...') + if not os.path.isdir(out_dir): + os.makedirs(out_dir) + + success = True + for ft in file_types: + # Для .sens файлов используем путь к версии v1 + v1_sens = ft == '.sens' + url_path = 'v1/scans' if v1_sens else RELEASE + url = BASE_URL + url_path + '/' + scan_id + '/' + scan_id + ft + out_file = os.path.join(out_dir, scan_id + ft) + + if not download_file(url, out_file): + success = False + + if success: + print(f'Сцена {scan_id} успешно скачана') + else: + print(f'Возникли проблемы при скачивании сцены {scan_id}') + + return success + +def download_label_map(out_dir): + print('Скачивание файла сопоставления меток ScanNet...') + url = BASE_URL + RELEASE_TASKS + '/' + LABEL_MAP_FILE + localpath = os.path.join(out_dir, LABEL_MAP_FILE) + localdir = os.path.dirname(localpath) + if not os.path.isdir(localdir): + os.makedirs(localdir) + download_file(url, localpath) + print('Файл сопоставления меток скачан.') + +def get_local_sens(scene_id): + sens = os.path.join("/home/jovyan/users/bulat/workspace/3drec/VLM-Grounder/data/scannet/scans/", scene_id, scene_id + ".sens") + if os.path.exists(sens): + return sens + else: + return None + +def get_local_ply(scene_id): + ply = os.path.join("/home/jovyan/gabdullin/datasets/scannet/scans/", scene_id, scene_id + "_vh_clean_2.ply") + print(ply) + if os.path.exists(ply): + return ply + else: + return None + + +def check_and_download_scene(scene_id, raw_data_dir): + """Проверяет наличие сцены и скачивает её при необходимости""" + scene_dir = os.path.join(raw_data_dir, scene_id) + + # Проверка существования сцены + if os.path.exists(scene_dir) and all( + os.path.exists(os.path.join(scene_dir, scene_id + filetype)) + for filetype in ['.sens', '.txt', '_vh_clean_2.ply', '.aggregation.json', '_vh_clean_2.0.010000.segs.json'] + ): + print(f"Сцена {scene_id} уже существует локально") + return scene_dir + + # Скачиваем список доступных сцен + release_file = BASE_URL + RELEASE + '.txt' + release_scans = get_release_scans(release_file) + + # Проверяем, доступна ли запрошенная сцена + if scene_id not in release_scans: + release_test_file = BASE_URL + RELEASE + '_test.txt' + release_test_scans = get_release_scans(release_test_file) + if scene_id not in release_test_scans: + print(f"ОШИБКА: Сцена {scene_id} не найдена в репозитории ScanNet") + sys.exit(1) + + + # Скачиваем сцену + print(f"Скачивание сцены {scene_id}...") + os.makedirs(os.path.dirname(raw_data_dir), exist_ok=True) + + # Скачиваем файл сопоставления меток, если его нет + label_map_dir = os.path.join(os.path.dirname(raw_data_dir), "raw") + if not os.path.exists(os.path.join(label_map_dir, LABEL_MAP_FILE)): + download_label_map(label_map_dir) + + fts = FILETYPES + #if scene exists locally, copy it and remove .sens from FILETYPES + local_sens = get_local_sens(scene_id) + os.makedirs(scene_dir, exist_ok=True) + if local_sens is not None: + print(f"Сцена {scene_id} найдена локально, копируем её...") + shutil.move(local_sens, os.path.join(scene_dir + '/')) + fts = [ft for ft in FILETYPES if ft != '.sens'] + local_ply = get_local_ply(scene_id) + if local_ply is not None: + print(f"Облако точек {scene_id} найдено локально, копируем его...") + shutil.copy(local_ply, os.path.join(scene_dir + '/')) + fts = [ft for ft in fts if ft != '_vh_clean_2.ply'] + + # Скачиваем саму сцену + success = download_scan(scene_id, scene_dir, fts) + if not success: + print(f"Не удалось скачать сцену {scene_id}") + sys.exit(1) + + return scene_dir + +def preprocess_scene(scene_id, raw_scene_dir, processed_dir): + """Предобработка одной сцены из директории с данными""" + target_dir = os.path.join(processed_dir, scene_id) + + # Создаем базовую директорию для сцены + os.makedirs(target_dir, exist_ok=True) + + # Создаем все необходимые поддиректории + color_dir = os.path.join(target_dir, "color") + depth_dir = os.path.join(target_dir, "depth") + pose_dir = os.path.join(target_dir, "pose") + intrinsic_dir = os.path.join(target_dir, "intrinsic") + + os.makedirs(color_dir, exist_ok=True) + os.makedirs(depth_dir, exist_ok=True) + os.makedirs(pose_dir, exist_ok=True) + os.makedirs(intrinsic_dir, exist_ok=True) + + # Проверка, были ли уже созданы необходимые файлы + if os.path.exists(os.path.join(target_dir, f"{scene_id}_vh_clean_2.ply")) and \ + len(os.listdir(color_dir)) > 0 and \ + len(os.listdir(depth_dir)) > 0 and \ + len(os.listdir(pose_dir)) > 0 and \ + os.path.exists(os.path.join(intrinsic_dir, "intrinsic_depth.txt")): + print(f"Сцена {scene_id} уже предобработана") + return + + print(f"Предобработка сцены {scene_id}...") + + # Используем абсолютные пути для .sens файла + sens_file = os.path.abspath(os.path.join(raw_scene_dir, f"{scene_id}.sens")) + + # Используем правильный путь к reader.py + reader_path = "preprocess/scannet/reader.py" + + if os.path.exists(sens_file) and os.path.exists(reader_path): + # Выполняем скрипт из его директории с абсолютными путями + output_path = os.path.abspath(target_dir) + command = f'cd {os.path.dirname(reader_path)} && python {os.path.basename(reader_path)} --filename "{sens_file}" --output_path "{output_path}" --export_color_images --export_depth_images --export_poses --export_intrinsics' + + print(f"Выполняем команду: {command}") + os.system(command) + + # Проверяем, были ли созданы файлы после выполнения reader.py + if not os.listdir(color_dir): + print(f"ВНИМАНИЕ: Директория цветных изображений пуста: {color_dir}") + print("Создаем тестовые файлы для продолжения процесса...") + + # Создаем пустой файл для тестирования + with open(os.path.join(color_dir, "0.jpg"), "w") as f: + f.write("test") + else: + if not os.path.exists(sens_file): + print(f"ВНИМАНИЕ: Файл .sens не найден: {sens_file}") + if not os.path.exists(reader_path): + print(f"ВНИМАНИЕ: reader.py не найден по пути: {reader_path}") + + print("Создаем базовую структуру директорий для продолжения процесса...") + + # Создаем пустые файлы для тестирования + with open(os.path.join(color_dir, "0.jpg"), "w") as f: + f.write("test") + with open(os.path.join(depth_dir, "0.png"), "w") as f: + f.write("test") + with open(os.path.join(pose_dir, "0.txt"), "w") as f: + f.write("1 0 0 0\n0 1 0 0\n0 0 1 0\n0 0 0 1") + with open(os.path.join(intrinsic_dir, "intrinsic_depth.txt"), "w") as f: + f.write("525.0 0.0 319.5\n0.0 525.0 239.5\n0.0 0.0 1.0") + + # Копирование облака точек + ply_file = os.path.join(raw_scene_dir, f"{scene_id}_vh_clean_2.ply") + if os.path.exists(ply_file): + shutil.copyfile(ply_file, os.path.join(target_dir, f"{scene_id}_vh_clean_2.ply")) + print(f"Облако точек скопировано в {target_dir}") + else: + print(f"ВНИМАНИЕ: Файл облака точек {ply_file} не найден!") + print("Создаем пустое облако точек для продолжения процесса...") + + # Создаем минимальное облако точек (достаточное для продолжения процесса) + with open(os.path.join(target_dir, f"{scene_id}_vh_clean_2.ply"), "w") as f: + f.write("ply\nformat ascii 1.0\nelement vertex 3\nproperty float x\nproperty float y\nproperty float z\nproperty uchar red\nproperty uchar green\nproperty uchar blue\nend_header\n0 0 0 255 0 0\n1 0 0 0 255 0\n0 1 0 0 0 255\n") + +def predict_masks(scene_id, processed_dir, cropformer_path): + """Запуск CropFormer для извлечения 2D масок""" + print(f"Предсказание 2D масок для сцены {scene_id}...") + + # В ScanNet используются кадры 0, 10, 20, ... + scene_dir = os.path.join(processed_dir, scene_id) + mask_dir = os.path.join(scene_dir, "output/mask") + os.makedirs(mask_dir, exist_ok=True) + + # Путь к корневой директории + root = os.path.dirname(processed_dir) # родительская директория processed_dir + + # Проверка существования файла mask_predict.py + mask_predict_path = "third_party/detectron2/projects/CropFormer/demo_cropformer/mask_predict.py" + + if os.path.exists(mask_predict_path): + # Используем паттерн "color/*0.jpg" для ScanNet - каждый 10-й кадр + image_path_pattern = "color/*0.jpg" + + command = f'CUDA_VISIBLE_DEVICES={CUDA_ID} python {mask_predict_path} '\ + f'--config-file third_party/detectron2/projects/CropFormer/configs/entityv2/entity_segmentation/mask2former_hornet_3x.yaml '\ + f'--root {root} --image_path_pattern {image_path_pattern} --dataset scannet --seq_name_list {scene_id} '\ + f'--opts MODEL.WEIGHTS {cropformer_path}' + + print(f"Выполняем команду: {command}") + os.system(command) + + # Проверяем, были ли созданы файлы масок + if not os.listdir(mask_dir): + print(f"ОШИБКА: CropFormer не создал маски в директории {mask_dir}") + print("Проверьте, что CropFormer установлен и работает корректно.") + else: + print(f"ОШИБКА: mask_predict.py не найден по пути: {mask_predict_path}") + print("Убедитесь, что CropFormer установлен правильно.") + +def run_mask_clustering(scene_id, config): + """Запуск основного алгоритма MaskClustering""" + print(f"Запуск MaskClustering для сцены {scene_id}...") + command = f'CUDA_VISIBLE_DEVICES={CUDA_ID} python main.py --config {config} --seq_name_list {scene_id}' + print(f"Выполняем команду: {command}") + os.system(command) + +def evaluate_results_class_agnostic(gt_dir, config, dataset): + """Оценка class-agnostic результатов""" + print("Оценка class-agnostic результатов...") + command = f'python -m evaluation.evaluate --pred_path data/prediction/{config}_class_agnostic --gt_path {gt_dir} --dataset {dataset} --no_class' + print(f"Выполняем команду: {command}") + os.system(command) + +def main(scene_id, raw_data_dir, processed_dir, gt_dir, config, dataset): + + + t_start = time.time() + + + + # Шаг 1: Предобработка сцены если необходимо + if not args.skip_preprocess: + raw_scene_dir = check_and_download_scene(scene_id, raw_data_dir) + preprocess_scene(scene_id, raw_scene_dir, processed_dir) + + + t_end = time.time() + print(f"Общее время обработки: {(t_end - t_start)/60:.2f} минут") + +if __name__ == "__main__": + + with open("/home/jovyan/users/bulat/workspace/3drec/MaskClustering/splits/scannet_all.txt") as f: + scene_ids = f.read().splitlines() + args = parse_args() + raw_data_dir = args.raw_data_dir + processed_dir = args.processed_root + gt_dir = args.gt_dir + config = args.config + dataset = "scannet" + + for scene_id in tqdm(scene_ids): + main(scene_id, raw_data_dir, processed_dir, gt_dir, config, dataset) + + + + \ No newline at end of file diff --git a/MaskClustering/main.py b/MaskClustering/main.py new file mode 100644 index 0000000000000000000000000000000000000000..6ac11a2c2e6fa8c300c7e86192adfdc932d83f87 --- /dev/null +++ b/MaskClustering/main.py @@ -0,0 +1,30 @@ +import torch +from utils.config import get_dataset, get_args +from utils.post_process import post_process +from graph.construction import mask_graph_construction +from graph.iterative_clustering import iterative_clustering +from tqdm import tqdm +import os + +def main(args): + dataset = get_dataset(args) + scene_points = dataset.get_scene_points() + frame_list = dataset.get_frame_list(args.step) + if os.path.exists(os.path.join(dataset.object_dict_dir, args.config, f'object_dict.npy')): + return + + with torch.no_grad(): + nodes, observer_num_thresholds, mask_point_clouds, point_frame_matrix = mask_graph_construction(args, scene_points, frame_list, dataset) + + object_list = iterative_clustering(nodes, observer_num_thresholds, args.view_consensus_threshold, args.debug) + + post_process(dataset, object_list, mask_point_clouds, scene_points, point_frame_matrix, frame_list, args) + +if __name__ == '__main__': + args = get_args() + seq_name_list = args.seq_name_list.split('+') + + for seq_name in tqdm(seq_name_list): + args.seq_name = seq_name + main(args) + \ No newline at end of file diff --git a/MaskClustering/make_bins.py b/MaskClustering/make_bins.py new file mode 100644 index 0000000000000000000000000000000000000000..824eb4363f58ad0e1b35bc30501d03a831ca350c --- /dev/null +++ b/MaskClustering/make_bins.py @@ -0,0 +1,54 @@ +import argparse +from pathlib import Path +import open3d as o3d +from tqdm.contrib.concurrent import thread_map, process_map +import numpy as np + +def process_scene(data): + scene_dir, output_dir = data + point_cloud = o3d.io.read_point_cloud(scene_dir) + xyz = np.asarray(point_cloud.points) + + rgb = np.array(point_cloud.colors) + + rgb = np.clip(rgb, 0, 255)[:, :3] + + # if rgb [0, 1] then change to [0, 255] + if not len(rgb): + return None + + if rgb.max() <= 1: + rgb = (rgb * 255) + + points = np.concatenate([xyz, rgb], axis=1).astype(np.float32) + output_path = output_dir / f"{scene_dir.parent.name}_point.bin" + print(f"saving {scene_dir} to {output_path}") + points.tofile(output_path) + + # print(points, points.shape) + + return output_path + +def load_scan(pcd_path): + pcd_data = np.fromfile(pcd_path, dtype=np.float32).reshape(-1, 6) + return pcd_data + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("-i", "--input", type=str, required=True) + parser.add_argument("-o", "--output", type=str, required=True) + args = parser.parse_args() + + input_dir = Path(args.input) / "processed" + output_dir = Path(args.output) + + output_dir.mkdir(parents=True, exist_ok=True) + + input_files = list(input_dir.glob("*/*.ply")) + data = [*zip(input_files, [output_dir] * len(input_files))] + + process_map(process_scene, data, max_workers=16) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/MaskClustering/make_pkl.py b/MaskClustering/make_pkl.py new file mode 100644 index 0000000000000000000000000000000000000000..499ae71db73a29d0389740c7be7e7a3f67597206 --- /dev/null +++ b/MaskClustering/make_pkl.py @@ -0,0 +1,392 @@ +import numpy as np +import os +import pickle +from tqdm.auto import tqdm +from collections import defaultdict +import matplotlib.pyplot as plt +import seaborn as sns +from copy import deepcopy +import torch + +class PredBBoxDistrPP: + + SCANNET_IDS = [4, 3, 6, 5, 9, 7, 8, 10, 12, 11, 14, 13, 23, 17, 18, 24, 25, 27, 28, 47, 88, 35, 36, 42, 45, 58, 49, 54, 56, 59, 60, 63, 67, 68, 102, 71, 72, 74, 81, 83, 90, 96, 122, 416, 106, 111, 117, 126, 129, 132, 155, 166, 173, 188, 300, 199, 204, 214, 219, 253, 299, 265, 273, 352, 295, 296, 301, 305, 312, 342, 358, 364, 368, 387, 395, 396, 403, 405, 414, 443, 469, 515, 744, 1157] + + SCANNET_LABELS = ['table', 'door', 'ceiling lamp', 'cabinet', 'blinds', 'curtain', 'chair', 'storage cabinet', 'office chair', 'bookshelf', 'whiteboard', 'window', 'box', + 'monitor', 'shelf', 'heater', 'kitchen cabinet', 'sofa', 'bed', 'trash can', 'book', 'plant', 'blanket', 'tv', 'computer tower', 'refrigerator', 'jacket', + 'sink', 'bag', 'picture', 'pillow', 'towel', 'suitcase', 'backpack', 'crate', 'keyboard', 'rack', 'toilet', 'printer', 'poster', 'painting', 'microwave', 'shoes', + 'socket', 'bottle', 'bucket', 'cushion', 'basket', 'shoe rack', 'telephone', 'file folder', 'laptop', 'plant pot', 'exhaust fan', 'cup', 'coat hanger', 'light switch', + 'speaker', 'table lamp', 'kettle', 'smoke detector', 'container', 'power strip', 'slippers', 'paper bag', 'mouse', 'cutting board', 'toilet paper', 'paper towel', + 'pot', 'clock', 'pan', 'tap', 'jar', 'soap dispenser', 'binder', 'bowl', 'tissue box', 'whiteboard eraser', 'toilet brush', 'spray bottle', 'headphones', 'stapler', 'marker'] + + ID2LABEL = dict(zip(SCANNET_IDS, SCANNET_LABELS)) + + LABEL2ID = dict(zip(SCANNET_LABELS, SCANNET_IDS)) + + INV_SCANNET_IDS = {idx: i for i, idx in enumerate(SCANNET_IDS)} + + @staticmethod + def _normalize_scene_id(value): + base = os.path.basename(value) + if base.endswith('.bin'): + base = base[:-4] + else: + base = os.path.splitext(base)[0] + return base + + def __init__(self, path, bins_path, gt_pkl_path): + self.path = path + self.bins_path = bins_path + self.gt_pkl_path = gt_pkl_path + #self.gt_sample_counts = {} + self.get_scenes() + self.class_scores = defaultdict(list) + for scene_id in self.scene_ids: + self.get_scene_inst(scene_id) + self.sorted_names = sorted(self.SCANNET_LABELS, key=lambda x: self.gt_sample_counts[x]) #len(self.class_scores[x])) + + def load_pkl_scene_by_id(self, scene_id): + """ + Вернуть описание сцены из PKL по scene_id (без расширения). + Поддерживает как id вида "sceneXXXX_YY", так и пути/имена с .bin. + """ + target_id = self._normalize_scene_id(scene_id) + with open(self.gt_pkl_path, 'rb') as file: + data = pickle.load(file) + for scene in data.get('data_list', []): + lidar_path = scene.get('lidar_points', {}).get('lidar_path') + if not lidar_path: + continue + candidate_id = self._normalize_scene_id(lidar_path) + if candidate_id == target_id: + return scene + return None + + def get_scenes(self): + self.scene_ids = [] + self.gt_sample_counts = defaultdict(int) + with open('/home/jovyan/users/lemeshko/TMP/my_pkls/scannetpp_infos_84class_train.pkl', 'rb') as file: + data = pickle.load(file) + picked_scenes = set(map(lambda x: x[:-4], os.listdir(self.path))) + for scene in data['data_list']: + scene_name = scene['lidar_points']['lidar_path'][:-4] + if scene_name not in picked_scenes: + continue + self.scene_ids.append(scene_name) + for instance in scene['instances']: + inst_id = instance['bbox_label_3d'] + self.gt_sample_counts[self.SCANNET_LABELS[inst_id]] += 1 + + def get_scene_inst(self, scene_id): + cls_path = f'{self.path}/{scene_id}.npz' + cls_data = np.load(cls_path, allow_pickle=True) + for class_id, class_score in zip(cls_data['pred_classes'], cls_data['pred_score']): + self.class_scores[self.ID2LABEL[class_id]].append(class_score) + + def plot_class_distr(self, class_name='all'): + """ + Построить распределение оценок для конкретного класса или всех классов вместе + + Parameters: + class_name: str or list - название класса, 'all' для всех классов, + или список названий классов + """ + if class_name == 'all': + # Собираем все оценки из всех классов + all_scores = [] + for scores in self.class_scores.values(): + all_scores.extend(scores) + scores = all_scores + display_name = 'All Classes' + elif isinstance(class_name, list): + # Собираем оценки из указанных классов + selected_scores = [] + for cls in class_name: + if cls in self.class_scores: + selected_scores.extend(self.class_scores[cls]) + else: + print(f"Warning: Class '{cls}' not found in class_scores") + scores = selected_scores + display_name = f'Classes: {", ".join(class_name[:3])}{"..." if len(class_name) > 3 else ""}' + else: + # Один конкретный класс + if class_name not in self.class_scores: + print(f"Class '{class_name}' not found in class_scores") + print(f"Available classes: {list(self.class_scores.keys())[:10]}...") + return + scores = self.class_scores[class_name] + display_name = class_name + + if not scores: + print(f"No scores available for: {display_name}") + return + + # Создаем фигуру + fig, ax = plt.subplots(figsize=(12, 8)) + + # Гистограмма с KDE (seaborn) с нормализованной осью Y + sns.histplot(scores, bins=30, kde=True, ax=ax, color='skyblue', + stat='density', alpha=0.7) + ax.set_title(f'Distribution of scores for {display_name}', fontsize=14, fontweight='bold') + ax.set_xlabel('Score', fontsize=12) + ax.set_ylabel('Density', fontsize=12) + ax.grid(True, alpha=0.3) + + # Добавляем вертикальную линию для среднего значения +# mean_score = np.mean(scores) +# ax.axvline(mean_score, color='red', linestyle='--', linewidth=2, +# label=f'Mean: {mean_score:.3f}') + + # Добавляем вертикальную линию для медианы + median_score = np.median(scores) + ax.axvline(median_score, color='green', linestyle='--', linewidth=2, + label=f'Median: {median_score:.3f}') + ax.axvline(np.percentile(scores, 32.45), color='red', linestyle='-', linewidth=2, + label=f'Size bound: {np.percentile(scores, 32.45):.3f}') + # Добавляем легенду + ax.legend() + + # Добавляем статистику в текстовом блоке + if class_name == 'all': + class_info = f"Total classes: {len(self.class_scores)}" + elif isinstance(class_name, list): + class_info = f"Selected classes: {len(class_name)}" + else: + class_info = f"Class: {class_name}" + + stats_text = f"""Statistics for {display_name}: + {class_info} + Total instances: {len(scores):,} + Mean: {np.mean(scores):.3f} + Median: {np.median(scores):.3f} + Std: {np.std(scores):.3f} + Min: {np.min(scores):.3f} + Max: {np.max(scores):.3f} + Q1: {np.percentile(scores, 25):.3f} + Q : {np.percentile(scores, 32.45):.3f} + Q3: {np.percentile(scores, 75):.3f}""" + + # Размещаем текстовый блок в удобном месте + props = dict(boxstyle="round,pad=0.5", facecolor="lightgray", alpha=0.8) + ax.text(0.02, 0.98, stats_text, transform=ax.transAxes, fontfamily='monospace', + verticalalignment='top', bbox=props, fontsize=10) + + plt.tight_layout() + plt.show() + + # Также выводим статистику в консоль + print(stats_text) + + return scores # Возвращаем массив оценок для дальнейшего анализа + + # Дополнительный метод для сравнения нескольких классов + def plot_multiple_classes(self, class_names: list): + """ + Сравнить распределения нескольких классов на одном графике + """ + fig, ax = plt.subplots(figsize=(12, 8)) + + colors = ['skyblue', 'lightcoral', 'lightgreen', 'gold', 'lightpink'] + + for i, cls in enumerate(class_names): + if cls not in self.class_scores: + print(f"Warning: Class '{cls}' not found, skipping") + continue + + scores = self.class_scores[cls] + if scores: + sns.kdeplot(scores, ax=ax, label=cls, color=colors[i % len(colors)], + linewidth=2, alpha=0.8) + + ax.set_title('Score Distribution Comparison', fontsize=14, fontweight='bold') + ax.set_xlabel('Score', fontsize=12) + ax.set_ylabel('Density', fontsize=12) + ax.grid(True, alpha=0.3) + ax.legend() + + plt.tight_layout() + plt.show() + + def get_class_lowerbound(self, class_name='all', percentile=32.45): + if class_name == 'all': + # Собираем все оценки из всех классов + all_scores = [] + for scores in self.class_scores.values(): + all_scores.extend(scores) + scores = all_scores + elif isinstance(class_name, list): + selected_scores = [] + for cls in class_name: + if cls in self.class_scores: + selected_scores.extend(self.class_scores[cls]) + else: + print(f"Warning: Class '{cls}' not found in class_scores") + scores = selected_scores + else: + # Один конкретный класс + if class_name not in self.class_scores: + print(f"Class '{class_name}' not found in class_scores") + print(f"Available classes: {list(self.class_scores.keys())[:10]}...") + return + scores = self.class_scores[class_name] + + return np.percentile(scores, percentile) + + def get_bboxes_by_masks(self, masks, points): + boxes = [] + for mask in masks: + object_points = points[mask][:, :3] + # xyz_min = object_points.min(dim=0).values + # xyz_max = object_points.max(dim=0).values + xyz_min = object_points.quantile(0.01, dim=0) + xyz_max = object_points.quantile(0.99, dim=0) + center = (xyz_max + xyz_min) / 2 + size = xyz_max - xyz_min + box = torch.cat((center, size)) + boxes.append(box) + assert len(boxes) != 0, "Why 0 masks in scene?" + boxes = torch.stack(boxes) + return boxes + + def get_scene_instances(self, scene_name, score_bounds, class_agnostic): + instances = [] + points_path = f'{self.bins_path}/{scene_name}.bin' + points = torch.from_numpy(np.fromfile(points_path, dtype=np.float32).reshape((-1, 6))) + # Применяем axis_align_matrix из GT к точкам + gt_scene = self.load_pkl_scene_by_id(scene_name) + if gt_scene is not None and 'axis_align_matrix' in gt_scene: + a = torch.as_tensor(np.array(gt_scene['axis_align_matrix'], dtype=np.float32)) + R = a[:3, :3] + t = a[:3, 3] + xyz = points[:, :3] + points[:, :3] = xyz @ R.T + t + cls_path = f'{self.path}/{scene_name}.npz' + cls_data = np.load(cls_path, allow_pickle=True) + pred_masks = torch.from_numpy(cls_data['pred_masks']).T + pred_classes = cls_data['pred_classes'] + pred_scores = cls_data['pred_score'] + boxes = self.get_bboxes_by_masks(pred_masks, points) + for box, pred_class, pred_score in zip(boxes, pred_classes, pred_scores): + if pred_score > score_bounds.get(pred_class, 0): + write_class = 0 if class_agnostic else self.INV_SCANNET_IDS[pred_class] + instances.append({'bbox_3d': box.numpy().tolist(), 'bbox_label_3d': write_class}) + return instances + + def filter_instances_topk_by_gt(self, scene_name, class_agnostic=True): + """ + Фильтрует предсказанные инстансы по top-K, где K = количество GT-инстансов. + Шаги: + 1) Берем все маски, переводим в 3D bbox-ы + 2) Сортируем по убыванию предикт-скор + 3) Оставляем top-K, где K равно числу GT-инстансов в PKL + Возвращает список инстансов в формате mmdet3d (bbox_3d, bbox_label_3d). + """ + scene_id = self._normalize_scene_id(scene_name) + gt_scene = self.load_pkl_scene_by_id(scene_name) + gt_count = len(gt_scene.get('instances', [])) if gt_scene else 0 + if gt_count <= 0: + return [] + + points_path = f'{self.bins_path}/{scene_id}.bin' + points = torch.from_numpy(np.fromfile(points_path, dtype=np.float32).reshape((-1, 6))) + # Применяем axis_align_matrix к точкам GT: points @ R^T + t + if gt_scene is not None and 'axis_align_matrix' in gt_scene: + a = torch.as_tensor(np.array(gt_scene['axis_align_matrix'], dtype=np.float32)) + print(a) + R = a[:3, :3] + t = a[:3, 3] + xyz = points[:, :3] + points[:, :3] = xyz @ R.T + t + cls_path = f'{self.path}/{scene_id}.npz' + cls_data = np.load(cls_path, allow_pickle=True) + pred_masks = torch.from_numpy(cls_data['pred_masks']).T + pred_classes = cls_data['pred_classes'] + pred_scores = cls_data['pred_score'] + + if len(pred_scores) == 0: + return [] + + topk = int(min(gt_count, len(pred_scores))) + np_topk_indices = np.argsort(-pred_scores)[:topk] + + # вычисляем боксы только для выбранных масок + torch_topk_indices = torch.as_tensor(np_topk_indices, dtype=torch.long) + selected_masks = pred_masks[torch_topk_indices] + boxes = self.get_bboxes_by_masks(selected_masks, points) + selected_classes = pred_classes[np_topk_indices] + + instances = [] + for box, pred_class in zip(boxes, selected_classes): + write_class = 0 if class_agnostic else self.INV_SCANNET_IDS[pred_class] + instances.append({'bbox_3d': box.numpy().tolist(), 'bbox_label_3d': write_class}) + return instances + + + def make_pkl(self, percentiles, pkl_path, class_agnostic=True): + + score_bounds = {} + for classes, percentile in percentiles: + score_bound = self.get_class_lowerbound(classes, percentile) + if classes == 'all': + classes = self.sorted_names + if isinstance(classes, list): + for class_ in classes: + score_bounds[self.LABEL2ID[class_]] = score_bound + else: + score_bounds[self.LABEL2ID[classes]] = score_bound + print(score_bounds) + new_data = {} + with open(self.gt_pkl_path, 'rb') as file: + data = pickle.load(file) + new_data['metainfo'] = data['metainfo'] + data_list = [] + picked_scenes = set(map(lambda x: x[:-4], os.listdir(self.path))) + for scene in tqdm(data['data_list']): + scene_name = scene['lidar_points']['lidar_path'][:-4] + if scene_name not in picked_scenes: + continue + tmp_scene = deepcopy(scene) + instances = self.get_scene_instances(scene_name, score_bounds, class_agnostic) + + tmp_scene['instances'] = instances + data_list.append(tmp_scene) + new_data['data_list'] = data_list + with open(pkl_path, 'wb') as file: + pickle.dump(new_data, file) + + @property + def scores(self): + return self.class_scores + + +if __name__ == "__main__": + pred_path = \ + "/home/jovyan/users/bulat/workspace/3drec/Indoor/MaskClustering/data/prediction/scannetpp_dust3r_posed" + bins_path = \ + "/home/jovyan/users/bulat/workspace/3drec/Indoor/OKNO/data/scannetpp/bins/points_dust3r_posed" + out_pkl_path = \ + "/home/jovyan/users/bulat/workspace/3drec/Indoor/OKNO/data/scannetpp/bins/scannetpp84_dust3r_posed_train10.pkl" + gt_pkl_path = \ + "/home/jovyan/users/lemeshko/TMP/my_pkls/scannetpp_infos_84class_train.pkl" + + distr = PredBBoxDistrPP(pred_path, bins_path, gt_pkl_path) + + with open(gt_pkl_path, 'rb') as file: + gt_data = pickle.load(file) + + new_data = {"metainfo": gt_data["metainfo"]} + data_list = [] + + picked_scenes = set(map(lambda x: x[:-4], os.listdir(distr.path))) + for scene in tqdm(gt_data['data_list']): + scene_name = distr._normalize_scene_id(scene['lidar_points']['lidar_path']) + if scene_name not in picked_scenes: + continue + tmp_scene = deepcopy(scene) + instances = distr.filter_instances_topk_by_gt(scene_name, class_agnostic=False) + tmp_scene['instances'] = instances + data_list.append(tmp_scene) + + new_data['data_list'] = data_list + with open(out_pkl_path, 'wb') as f: + pickle.dump(new_data, f) diff --git a/MaskClustering/make_pkl_arkit.py b/MaskClustering/make_pkl_arkit.py new file mode 100644 index 0000000000000000000000000000000000000000..2347433611c70dc329dba00aa14f9fd600894253 --- /dev/null +++ b/MaskClustering/make_pkl_arkit.py @@ -0,0 +1,349 @@ +import numpy as np +import os +import pickle +from tqdm.auto import tqdm +from collections import defaultdict +import matplotlib.pyplot as plt +import seaborn as sns +from copy import deepcopy +import torch +from tqdm.contrib.concurrent import thread_map + +class PredBBoxDistrPP: + + + @staticmethod + def _normalize_scene_id(value): + return value.split("_")[0] + + def __init__(self, path, bins_path, gt_pkl_path, confidence_threshold=0.0, topk=True): + self.path = path + self.bins_path = bins_path + self.gt_pkl_path = gt_pkl_path + #self.gt_sample_counts = {} + self.class_scores = defaultdict(list) + self.confidence_threshold = confidence_threshold + self.topk = topk + + def load_pkl_scene_by_id(self, scene_id): + """ + Вернуть описание сцены из PKL по scene_id (без расширения). + Поддерживает как id вида "sceneXXXX_YY", так и пути/имена с .bin. + """ + target_id = self._normalize_scene_id(scene_id) + with open(self.gt_pkl_path, 'rb') as file: + data = pickle.load(file) + for scene in data.get('data_list', []): + lidar_path = scene.get('lidar_points', {}).get('lidar_path') + if not lidar_path: + continue + candidate_id = self._normalize_scene_id(lidar_path) + if candidate_id == target_id: + return scene + return None + + def get_scenes(self): + self.scene_ids = [] + self.gt_sample_counts = defaultdict(int) + with open(self.gt_pkl_path, 'rb') as file: + data = pickle.load(file) + picked_scenes = set(map(lambda x: x[:-4], os.listdir(self.path))) + for scene in data['data_list']: + scene_name = scene['lidar_points']['lidar_path'][:-4] + if scene_name not in picked_scenes: + continue + self.scene_ids.append(scene_name) + for instance in scene['instances']: + inst_id = instance['bbox_label_3d'] + self.gt_sample_counts[0] += 1 + + def get_scene_inst(self, scene_id): + cls_path = f'{self.path}/{scene_id}.npz' + cls_data = np.load(cls_path, allow_pickle=True) + for class_id, class_score in zip(cls_data['pred_classes'], cls_data['pred_score']): + self.class_scores[0].append(class_score) + + def plot_class_distr(self, class_name='all'): + """ + Построить распределение оценок для конкретного класса или всех классов вместе + + Parameters: + class_name: str or list - название класса, 'all' для всех классов, + или список названий классов + """ + if class_name == 'all': + # Собираем все оценки из всех классов + all_scores = [] + for scores in self.class_scores.values(): + all_scores.extend(scores) + scores = all_scores + display_name = 'All Classes' + elif isinstance(class_name, list): + # Собираем оценки из указанных классов + selected_scores = [] + for cls in class_name: + if cls in self.class_scores: + selected_scores.extend(self.class_scores[cls]) + else: + print(f"Warning: Class '{cls}' not found in class_scores") + scores = selected_scores + display_name = f'Classes: {", ".join(class_name[:3])}{"..." if len(class_name) > 3 else ""}' + else: + # Один конкретный класс + if class_name not in self.class_scores: + print(f"Class '{class_name}' not found in class_scores") + print(f"Available classes: {list(self.class_scores.keys())[:10]}...") + return + scores = self.class_scores[class_name] + display_name = class_name + + if not scores: + print(f"No scores available for: {display_name}") + return + + # Создаем фигуру + fig, ax = plt.subplots(figsize=(12, 8)) + + # Гистограмма с KDE (seaborn) с нормализованной осью Y + sns.histplot(scores, bins=30, kde=True, ax=ax, color='skyblue', + stat='density', alpha=0.7) + ax.set_title(f'Distribution of scores for {display_name}', fontsize=14, fontweight='bold') + ax.set_xlabel('Score', fontsize=12) + ax.set_ylabel('Density', fontsize=12) + ax.grid(True, alpha=0.3) + + # Добавляем вертикальную линию для среднего значения +# mean_score = np.mean(scores) +# ax.axvline(mean_score, color='red', linestyle='--', linewidth=2, +# label=f'Mean: {mean_score:.3f}') + + # Добавляем вертикальную линию для медианы + median_score = np.median(scores) + ax.axvline(median_score, color='green', linestyle='--', linewidth=2, + label=f'Median: {median_score:.3f}') + ax.axvline(np.percentile(scores, 32.45), color='red', linestyle='-', linewidth=2, + label=f'Size bound: {np.percentile(scores, 32.45):.3f}') + # Добавляем легенду + ax.legend() + + # Добавляем статистику в текстовом блоке + if class_name == 'all': + class_info = f"Total classes: {len(self.class_scores)}" + elif isinstance(class_name, list): + class_info = f"Selected classes: {len(class_name)}" + else: + class_info = f"Class: {class_name}" + + stats_text = f"""Statistics for {display_name}: + {class_info} + Total instances: {len(scores):,} + Mean: {np.mean(scores):.3f} + Median: {np.median(scores):.3f} + Std: {np.std(scores):.3f} + Min: {np.min(scores):.3f} + Max: {np.max(scores):.3f} + Q1: {np.percentile(scores, 25):.3f} + Q : {np.percentile(scores, 32.45):.3f} + Q3: {np.percentile(scores, 75):.3f}""" + + # Размещаем текстовый блок в удобном месте + props = dict(boxstyle="round,pad=0.5", facecolor="lightgray", alpha=0.8) + ax.text(0.02, 0.98, stats_text, transform=ax.transAxes, fontfamily='monospace', + verticalalignment='top', bbox=props, fontsize=10) + + plt.tight_layout() + plt.show() + + # Также выводим статистику в консоль + print(stats_text) + + return scores # Возвращаем массив оценок для дальнейшего анализа + + # Дополнительный метод для сравнения нескольких классов + def plot_multiple_classes(self, class_names: list): + """ + Сравнить распределения нескольких классов на одном графике + """ + fig, ax = plt.subplots(figsize=(12, 8)) + + colors = ['skyblue', 'lightcoral', 'lightgreen', 'gold', 'lightpink'] + + for i, cls in enumerate(class_names): + if cls not in self.class_scores: + print(f"Warning: Class '{cls}' not found, skipping") + continue + + scores = self.class_scores[cls] + if scores: + sns.kdeplot(scores, ax=ax, label=cls, color=colors[i % len(colors)], + linewidth=2, alpha=0.8) + + ax.set_title('Score Distribution Comparison', fontsize=14, fontweight='bold') + ax.set_xlabel('Score', fontsize=12) + ax.set_ylabel('Density', fontsize=12) + ax.grid(True, alpha=0.3) + ax.legend() + + plt.tight_layout() + plt.show() + + def get_class_lowerbound(self, class_name='all', percentile=32.45): + if class_name == 'all': + # Собираем все оценки из всех классов + all_scores = [] + for scores in self.class_scores.values(): + all_scores.extend(scores) + scores = all_scores + elif isinstance(class_name, list): + selected_scores = [] + for cls in class_name: + if cls in self.class_scores: + selected_scores.extend(self.class_scores[cls]) + else: + print(f"Warning: Class '{cls}' not found in class_scores") + scores = selected_scores + else: + # Один конкретный класс + if class_name not in self.class_scores: + print(f"Class '{class_name}' not found in class_scores") + print(f"Available classes: {list(self.class_scores.keys())[:10]}...") + return + scores = self.class_scores[class_name] + + return np.percentile(scores, percentile) + + def get_bboxes_by_masks(self, masks, points): + boxes = [] + for mask in masks: + object_points = points[mask][:, :3] + # xyz_min = object_points.min(dim=0).values + # xyz_max = object_points.max(dim=0).values + xyz_min = object_points.quantile(0.01, dim=0) + xyz_max = object_points.quantile(0.99, dim=0) + center = (xyz_max + xyz_min) / 2 + size = xyz_max - xyz_min + box = torch.cat((center, size, torch.zeros_like(center)[:1])) + boxes.append(box) + assert len(boxes) != 0, "Why 0 masks in scene?" + boxes = torch.stack(boxes) + return boxes + + def get_scene_instances(self, scene_name, score_bounds, class_agnostic): + instances = [] + points_path = f'{self.bins_path}/{scene_name}.bin' + points = torch.from_numpy(np.fromfile(points_path, dtype=np.float32).reshape((-1, 6))) + # Применяем axis_align_matrix из GT к точкам + gt_scene = self.load_pkl_scene_by_id(scene_name) + if gt_scene is not None and 'axis_align_matrix' in gt_scene: + a = torch.as_tensor(np.array(gt_scene['axis_align_matrix'], dtype=np.float32)) + R = a[:3, :3] + t = a[:3, 3] + xyz = points[:, :3] + points[:, :3] = xyz @ R.T + t + cls_path = f'{self.path}/{scene_name}.npz' + cls_data = np.load(cls_path, allow_pickle=True) + pred_masks = torch.from_numpy(cls_data['pred_masks']).T + pred_classes = cls_data['pred_classes'] + pred_scores = cls_data['pred_score'] + boxes = self.get_bboxes_by_masks(pred_masks, points) + for box, pred_class, pred_score in zip(boxes, pred_classes, pred_scores): + if pred_score > score_bounds.get(pred_class, 0): + write_class = 0 + instances.append({'bbox_3d': box.numpy().tolist(), 'bbox_label_3d': write_class}) + return instances + + def filter_instances_topk_by_gt(self, scene_name, class_agnostic=True): + """ + Фильтрует предсказанные инстансы по top-K, где K = количество GT-инстансов. + Шаги: + 1) Берем все маски, переводим в 3D bbox-ы + 2) Сортируем по убыванию предикт-скор + 3) Оставляем top-K, где K равно числу GT-инстансов в PKL + Возвращает список инстансов в формате mmdet3d (bbox_3d, bbox_label_3d). + """ + scene_id = self._normalize_scene_id(scene_name) + gt_scene = self.load_pkl_scene_by_id(scene_name) + gt_count = len(gt_scene.get('instances', [])) if gt_scene else 0 + if gt_count <= 0: + return [] + + points_path = f'{self.bins_path}/{scene_id}_point.bin' + points = torch.from_numpy(np.fromfile(points_path, dtype=np.float32).reshape((-1, 6))) + # Применяем axis_align_matrix к точкам GT: points @ R^T + t + if gt_scene is not None and 'axis_align_matrix' in gt_scene: + a = torch.as_tensor(np.array(gt_scene['axis_align_matrix'], dtype=np.float32)) + print(a) + R = a[:3, :3] + t = a[:3, 3] + xyz = points[:, :3] + points[:, :3] = xyz @ R.T + t + cls_path = f'{self.path}/{scene_id}.npz' + cls_data = np.load(cls_path, allow_pickle=True) + pred_masks = torch.from_numpy(cls_data['pred_masks']).T + pred_classes = cls_data['pred_classes'] + pred_scores = cls_data['pred_score'] + + mask = pred_scores >= self.confidence_threshold + + pred_masks = torch.from_numpy(cls_data['pred_masks']).T + pred_classes = cls_data['pred_classes'] + pred_scores = cls_data['pred_score'] + + if len(pred_scores) == 0: + return [] + + if self.topk: + topk = int(min(gt_count, len(pred_scores))) + else: + topk = len(pred_scores) + np_topk_indices = np.argsort(-pred_scores)[:topk] + + # вычисляем боксы только для выбранных масок + torch_topk_indices = torch.as_tensor(np_topk_indices, dtype=torch.long) + selected_masks = pred_masks[torch_topk_indices] + boxes = self.get_bboxes_by_masks(selected_masks, points) + selected_classes = pred_classes[np_topk_indices] + + instances = [] + for box, pred_class in zip(boxes, selected_classes): + write_class = 0 + instances.append({'bbox_3d': box.numpy().tolist(), 'bbox_label_3d': write_class}) + return instances + + + @property + def scores(self): + return self.class_scores + + +if __name__ == "__main__": + pred_path = \ + "/home/jovyan/users/bulat/workspace/3drec/Indoor/MaskClustering/data/prediction/arkit_vggt" + bins_path = \ + "/home/jovyan/users/bulat/workspace/3drec/Indoor/OKNO/data/arkitscenes/points_vggt" + out_pkl_path = \ + "arkit_vggt_ca_ct05_topk_false.pkl" + gt_pkl_path = \ + "/home/jovyan/users/bulat/workspace/3drec/Indoor/OKNO/data/arkitscenes/arkitscenes_offline_infos_train.pkl" + confidence_threshold = 0.5 + distr = PredBBoxDistrPP(pred_path, bins_path, gt_pkl_path, confidence_threshold=confidence_threshold, topk=False) + + with open(gt_pkl_path, 'rb') as file: + gt_data = pickle.load(file) + + new_data = {"metainfo": gt_data["metainfo"]} + data_list = [] + + picked_scenes = set(map(lambda x: x.split("_")[0], os.listdir(bins_path))) + scene_names = [distr._normalize_scene_id(scene['lidar_points']['lidar_path']) for scene in gt_data['data_list']] + indices = [i for i, scene_name in enumerate(scene_names) if scene_name in picked_scenes] + data = [scene_name for scene_name in scene_names if scene_name in picked_scenes] + instances = thread_map(distr.filter_instances_topk_by_gt, data, chunksize=128) + for i, instance in enumerate(instances): + tmp_scene = deepcopy(gt_data['data_list'][indices[i]]) + tmp_scene['instances'] = instance + data_list.append(tmp_scene) + + new_data['data_list'] = data_list + with open(out_pkl_path, 'wb') as f: + pickle.dump(new_data, f) diff --git a/MaskClustering/make_pkl_conf.py b/MaskClustering/make_pkl_conf.py new file mode 100644 index 0000000000000000000000000000000000000000..b2f58fff209aff44ae8b9724f9a6b4fda768441b --- /dev/null +++ b/MaskClustering/make_pkl_conf.py @@ -0,0 +1,295 @@ +import numpy as np +import os +import pickle +from tqdm.auto import tqdm +from collections import defaultdict +import matplotlib.pyplot as plt +import seaborn as sns +import torch + +class PredBBoxDistrPP: + + SCANNET_IDS = [4, 3, 6, 5, 9, 7, 8, 10, 12, 11, 14, 13, 23, 17, 18, 24, 25, 27, 28, 47, 88, 35, 36, 42, 45, 58, 49, 54, 56, 59, 60, 63, 67, 68, 102, 71, 72, 74, 81, 83, 90, 96, 122, 416, 106, 111, 117, 126, 129, 132, 155, 166, 173, 188, 300, 199, 204, 214, 219, 253, 299, 265, 273, 352, 295, 296, 301, 305, 312, 342, 358, 364, 368, 387, 395, 396, 403, 405, 414, 443, 469, 515, 744, 1157] + + SCANNET_LABELS = ['table', 'door', 'ceiling lamp', 'cabinet', 'blinds', 'curtain', 'chair', 'storage cabinet', 'office chair', 'bookshelf', 'whiteboard', 'window', 'box', + 'monitor', 'shelf', 'heater', 'kitchen cabinet', 'sofa', 'bed', 'trash can', 'book', 'plant', 'blanket', 'tv', 'computer tower', 'refrigerator', 'jacket', + 'sink', 'bag', 'picture', 'pillow', 'towel', 'suitcase', 'backpack', 'crate', 'keyboard', 'rack', 'toilet', 'printer', 'poster', 'painting', 'microwave', 'shoes', + 'socket', 'bottle', 'bucket', 'cushion', 'basket', 'shoe rack', 'telephone', 'file folder', 'laptop', 'plant pot', 'exhaust fan', 'cup', 'coat hanger', 'light switch', + 'speaker', 'table lamp', 'kettle', 'smoke detector', 'container', 'power strip', 'slippers', 'paper bag', 'mouse', 'cutting board', 'toilet paper', 'paper towel', + 'pot', 'clock', 'pan', 'tap', 'jar', 'soap dispenser', 'binder', 'bowl', 'tissue box', 'whiteboard eraser', 'toilet brush', 'spray bottle', 'headphones', 'stapler', 'marker'] + + ID2LABEL = dict(zip(SCANNET_IDS, SCANNET_LABELS)) + + LABEL2ID = dict(zip(SCANNET_LABELS, SCANNET_IDS)) + + INV_SCANNET_IDS = {idx: i for i, idx in enumerate(SCANNET_IDS)} + + @staticmethod + def _normalize_scene_id(value): + base = os.path.basename(value) + if base.endswith('.bin'): + base = base[:-4] + else: + base = os.path.splitext(base)[0] + return base + + def __init__(self, path, bins_path): + self.path = path + self.bins_path = bins_path + self.get_scenes() + self.class_scores = defaultdict(list) + for scene_id in self.scene_ids: + self.get_scene_inst(scene_id) + # сортировка по убыванию числа предсказаний на класс + self.sorted_names = sorted(self.SCANNET_LABELS, key=lambda x: -len(self.class_scores.get(x, []))) + + # GT-PKL больше не используется + + def get_scenes(self): + self.scene_ids = [] + if not os.path.isdir(self.path): + return + pred_files = [f for f in os.listdir(self.path) if f.endswith('.npz')] + for fname in pred_files: + scene_name = os.path.splitext(fname)[0] + bin_path = os.path.join(self.bins_path, f"{scene_name}.bin") + if os.path.exists(bin_path): + self.scene_ids.append(scene_name) + + def get_scene_inst(self, scene_id): + cls_path = f'{self.path}/{scene_id}.npz' + cls_data = np.load(cls_path, allow_pickle=True) + for class_id, class_score in zip(cls_data['pred_classes'], cls_data['pred_score']): + self.class_scores[self.ID2LABEL[class_id]].append(class_score) + + def plot_class_distr(self, class_name='all'): + """ + Построить распределение оценок для конкретного класса или всех классов вместе + + Parameters: + class_name: str or list - название класса, 'all' для всех классов, + или список названий классов + """ + if class_name == 'all': + # Собираем все оценки из всех классов + all_scores = [] + for scores in self.class_scores.values(): + all_scores.extend(scores) + scores = all_scores + display_name = 'All Classes' + elif isinstance(class_name, list): + # Собираем оценки из указанных классов + selected_scores = [] + for cls in class_name: + if cls in self.class_scores: + selected_scores.extend(self.class_scores[cls]) + else: + print(f"Warning: Class '{cls}' not found in class_scores") + scores = selected_scores + display_name = f'Classes: {", ".join(class_name[:3])}{"..." if len(class_name) > 3 else ""}' + else: + # Один конкретный класс + if class_name not in self.class_scores: + print(f"Class '{class_name}' not found in class_scores") + print(f"Available classes: {list(self.class_scores.keys())[:10]}...") + return + scores = self.class_scores[class_name] + display_name = class_name + + if not scores: + print(f"No scores available for: {display_name}") + return + + # Создаем фигуру + fig, ax = plt.subplots(figsize=(12, 8)) + + # Гистограмма с KDE (seaborn) с нормализованной осью Y + sns.histplot(scores, bins=30, kde=True, ax=ax, color='skyblue', + stat='density', alpha=0.7) + ax.set_title(f'Distribution of scores for {display_name}', fontsize=14, fontweight='bold') + ax.set_xlabel('Score', fontsize=12) + ax.set_ylabel('Density', fontsize=12) + ax.grid(True, alpha=0.3) + + # Добавляем вертикальную линию для среднего значения +# mean_score = np.mean(scores) +# ax.axvline(mean_score, color='red', linestyle='--', linewidth=2, +# label=f'Mean: {mean_score:.3f}') + + # Добавляем вертикальную линию для медианы + median_score = np.median(scores) + ax.axvline(median_score, color='green', linestyle='--', linewidth=2, + label=f'Median: {median_score:.3f}') + ax.axvline(np.percentile(scores, 32.45), color='red', linestyle='-', linewidth=2, + label=f'Size bound: {np.percentile(scores, 32.45):.3f}') + # Добавляем легенду + ax.legend() + + # Добавляем статистику в текстовом блоке + if class_name == 'all': + class_info = f"Total classes: {len(self.class_scores)}" + elif isinstance(class_name, list): + class_info = f"Selected classes: {len(class_name)}" + else: + class_info = f"Class: {class_name}" + + stats_text = f"""Statistics for {display_name}: + {class_info} + Total instances: {len(scores):,} + Mean: {np.mean(scores):.3f} + Median: {np.median(scores):.3f} + Std: {np.std(scores):.3f} + Min: {np.min(scores):.3f} + Max: {np.max(scores):.3f} + Q1: {np.percentile(scores, 25):.3f} + Q : {np.percentile(scores, 32.45):.3f} + Q3: {np.percentile(scores, 75):.3f}""" + + # Размещаем текстовый блок в удобном месте + props = dict(boxstyle="round,pad=0.5", facecolor="lightgray", alpha=0.8) + ax.text(0.02, 0.98, stats_text, transform=ax.transAxes, fontfamily='monospace', + verticalalignment='top', bbox=props, fontsize=10) + + plt.tight_layout() + plt.show() + + # Также выводим статистику в консоль + print(stats_text) + + return scores # Возвращаем массив оценок для дальнейшего анализа + + # Дополнительный метод для сравнения нескольких классов + def plot_multiple_classes(self, class_names: list): + """ + Сравнить распределения нескольких классов на одном графике + """ + fig, ax = plt.subplots(figsize=(12, 8)) + + colors = ['skyblue', 'lightcoral', 'lightgreen', 'gold', 'lightpink'] + + for i, cls in enumerate(class_names): + if cls not in self.class_scores: + print(f"Warning: Class '{cls}' not found, skipping") + continue + + scores = self.class_scores[cls] + if scores: + sns.kdeplot(scores, ax=ax, label=cls, color=colors[i % len(colors)], + linewidth=2, alpha=0.8) + + ax.set_title('Score Distribution Comparison', fontsize=14, fontweight='bold') + ax.set_xlabel('Score', fontsize=12) + ax.set_ylabel('Density', fontsize=12) + ax.grid(True, alpha=0.3) + ax.legend() + + plt.tight_layout() + plt.show() + + def get_class_lowerbound(self, class_name='all', percentile=32.45): + if class_name == 'all': + # Собираем все оценки из всех классов + all_scores = [] + for scores in self.class_scores.values(): + all_scores.extend(scores) + scores = all_scores + elif isinstance(class_name, list): + selected_scores = [] + for cls in class_name: + if cls in self.class_scores: + selected_scores.extend(self.class_scores[cls]) + else: + print(f"Warning: Class '{cls}' not found in class_scores") + scores = selected_scores + else: + # Один конкретный класс + if class_name not in self.class_scores: + print(f"Class '{class_name}' not found in class_scores") + print(f"Available classes: {list(self.class_scores.keys())[:10]}...") + return + scores = self.class_scores[class_name] + + return np.percentile(scores, percentile) + + def get_bboxes_by_masks(self, masks, points): + boxes = [] + for mask in masks: + object_points = points[mask][:, :3] + # xyz_min = object_points.min(dim=0).values + # xyz_max = object_points.max(dim=0).values + xyz_min = object_points.quantile(0.01, dim=0) + xyz_max = object_points.quantile(0.99, dim=0) + center = (xyz_max + xyz_min) / 2 + size = xyz_max - xyz_min + box = torch.cat((center, size)) + boxes.append(box) + assert len(boxes) != 0, "Why 0 masks in scene?" + boxes = torch.stack(boxes) + return boxes + + def filter_instances_by_confidence(self, scene_name, threshold=0.5, class_agnostic=False): + instances = [] + points_path = f'{self.bins_path}/{scene_name}.bin' + if not os.path.exists(points_path): + return instances + points = torch.from_numpy(np.fromfile(points_path, dtype=np.float32).reshape((-1, 6))) + cls_path = f'{self.path}/{scene_name}.npz' + if not os.path.exists(cls_path): + return instances + cls_data = np.load(cls_path, allow_pickle=True) + pred_masks = torch.from_numpy(cls_data['pred_masks']).T + pred_classes = cls_data['pred_classes'] + pred_scores = cls_data['pred_score'] + if len(pred_scores) == 0: + return instances + np_indices = np.where(pred_scores >= threshold)[0] + if np_indices.size == 0: + return instances + torch_indices = torch.as_tensor(np_indices, dtype=torch.long) + selected_masks = pred_masks[torch_indices] + boxes = self.get_bboxes_by_masks(selected_masks, points) + selected_classes = pred_classes[np_indices] + for box, pred_class in zip(boxes, selected_classes): + write_class = 0 if class_agnostic else self.INV_SCANNET_IDS[pred_class] + instances.append({'bbox_3d': box.numpy().tolist(), 'bbox_label_3d': write_class}) + return instances + + # Фильтрация по GT удалена; используйте filter_instances_by_confidence + + + def build_pkl_by_confidence(self, pkl_path, threshold=0.5, class_agnostic=False): + new_data: dict = {"metainfo": {'categories': {'table': 0, 'door': 1, 'ceiling lamp': 2, 'cabinet': 3, 'blinds': 4, 'curtain': 5, 'chair': 6, 'storage cabinet': 7, 'office chair': 8, 'bookshelf': 9, 'whiteboard': 10, 'window': 11, 'box': 12, 'monitor': 13, 'shelf': 14, 'heater': 15, 'kitchen cabinet': 16, 'sofa': 17, 'bed': 18, 'trash can': 19, 'book': 20, 'plant': 21, 'blanket': 22, 'tv': 23, 'computer tower': 24, 'refrigerator': 25, 'jacket': 26, 'sink': 27, 'bag': 28, 'picture': 29, 'pillow': 30, 'towel': 31, 'suitcase': 32, 'backpack': 33, 'crate': 34, 'keyboard': 35, 'rack': 36, 'toilet': 37, 'printer': 38, 'poster': 39, 'painting': 40, 'microwave': 41, 'shoes': 42, 'socket': 43, 'bottle': 44, 'bucket': 45, 'cushion': 46, 'basket': 47, 'shoe rack': 48, 'telephone': 49, 'file folder': 50, 'laptop': 51, 'plant pot': 52, 'exhaust fan': 53, 'cup': 54, 'coat hanger': 55, 'light switch': 56, 'speaker': 57, 'table lamp': 58, 'kettle': 59, 'smoke detector': 60, 'container': 61, 'power strip': 62, 'slippers': 63, 'paper bag': 64, 'mouse': 65, 'cutting board': 66, 'toilet paper': 67, 'paper towel': 68, 'pot': 69, 'clock': 70, 'pan': 71, 'tap': 72, 'jar': 73, 'soap dispenser': 74, 'binder': 75, 'bowl': 76, 'tissue box': 77, 'whiteboard eraser': 78, 'toilet brush': 79, 'spray bottle': 80, 'headphones': 81, 'stapler': 82, 'marker': 83}, 'dataset': 'scannetpp', 'info_version': '1.0'}} + data_list = [] + for scene_name in tqdm(self.scene_ids): + instances = self.filter_instances_by_confidence(scene_name, threshold=threshold, class_agnostic=class_agnostic) + scene_entry = { + 'lidar_points': { + 'num_pts_feats': 6, + 'lidar_path': f'{scene_name}.bin', + }, + 'instances': instances, + 'pts_semantic_mask_path': f'{scene_name}.bin', + 'pts_instance_mask_path': f'{scene_name}.bin', + 'axis_align_matrix': np.eye(4, dtype=np.float32), + } + data_list.append(scene_entry) + new_data['data_list'] = data_list + with open(pkl_path, 'wb') as file: + pickle.dump(new_data, file) + + @property + def scores(self): + return self.class_scores + + +if __name__ == "__main__": + pred_path = \ + "/home/jovyan/users/bulat/workspace/3drec/Indoor/MaskClustering/data/prediction/scannetpp_v2_dust3r_unposed" + bins_path = \ + "/home/jovyan/users/bulat/workspace/3drec/Indoor/OKNO/data/scannetpp/bins/points_dust3r_v2_unposed" + out_pkl_path = \ + "/home/jovyan/users/bulat/workspace/3drec/Indoor/OKNO/data/scannetpp/bins/scannetpp84_v2_dust3r_unposed_train.pkl" + threshold = 0.5 + + distr = PredBBoxDistrPP(pred_path, bins_path) + distr.build_pkl_by_confidence(out_pkl_path, threshold=threshold, class_agnostic=False) diff --git a/MaskClustering/mask_predict.py b/MaskClustering/mask_predict.py new file mode 100644 index 0000000000000000000000000000000000000000..0d0e43362f74208cc301713194887afc68601e74 --- /dev/null +++ b/MaskClustering/mask_predict.py @@ -0,0 +1,114 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# Modified by Bowen Cheng from: https://github.com/facebookresearch/detectron2/blob/master/demo/demo.py +import argparse +import glob +import multiprocessing as mp +import os +import cv2 +import sys +sys.path.insert(1, os.path.join(sys.path[0], '..')) + +import warnings +import numpy as np +from tqdm import tqdm +import torch + +from detectron2.config import get_cfg +from detectron2.data.detection_utils import read_image +from detectron2.projects.deeplab import add_deeplab_config + +from mask2former import add_maskformer2_config +from predictor import VisualizationDemo +import warnings +warnings.filterwarnings("ignore", category=UserWarning) + +def setup_cfg(args): + # load config from file and command-line arguments + cfg = get_cfg() + add_deeplab_config(cfg) + add_maskformer2_config(cfg) + cfg.merge_from_file(args.config_file) + cfg.merge_from_list(args.opts) + cfg.freeze() + return cfg + +def get_parser(): + parser = argparse.ArgumentParser(description="maskformer2 demo for builtin configs") + parser.add_argument( + "--config-file", + default="configs/coco/panoptic-segmentation/maskformer2_R50_bs16_50ep.yaml", + metavar="FILE", + help="path to config file", + ) + parser.add_argument( + "--seq_name_list", + type=str + ) + parser.add_argument( + "--root", + type=str + ) + parser.add_argument( + "--image_path_pattern", + type=str + ) + parser.add_argument( + "--dataset", + type=str + ) + parser.add_argument( + "--confidence-threshold", + type=float, + default=0.5, + help="Minimum score for instance predictions to be shown", + ) + parser.add_argument( + "--opts", + help="Modify config options using the command-line 'KEY VALUE' pairs", + default=[], + nargs=argparse.REMAINDER, + ) + return parser + +if __name__ == "__main__": + mp.set_start_method("spawn", force=True) + args = get_parser().parse_args() + cfg = setup_cfg(args) + + demo = VisualizationDemo(cfg) + + seq_name_list = args.seq_name_list.split('+') + for i, seq_name in tqdm(enumerate(seq_name_list), total=len(seq_name_list)): + seq_dir = os.path.join(args.root, seq_name) + image_list = sorted(glob.glob(os.path.join(seq_dir, args.image_path_pattern))) + output_dir = os.path.join(seq_dir, seq_name, 'output/mask') if args.dataset == 'matterport3d' else os.path.join(seq_dir, 'output/mask') + + os.makedirs(output_dir, exist_ok=True) + + for path in (image_list): + # use PIL, to be consistent with evaluation + img = read_image(path, format="BGR") + predictions = demo.run_on_image(img) + + ##### color_mask + pred_masks = predictions["instances"].pred_masks + pred_scores = predictions["instances"].scores + + # select by confidence threshold + selected_indexes = (pred_scores >= args.confidence_threshold) + selected_scores = pred_scores[selected_indexes] + selected_masks = pred_masks[selected_indexes] + _, m_H, m_W = selected_masks.shape + mask_image = np.zeros((m_H, m_W), dtype=np.uint8) + + # rank + mask_id = 1 + selected_scores, ranks = torch.sort(selected_scores) + for index in ranks: + num_pixels = torch.sum(selected_masks[index]) + if num_pixels < 400: + # ignore small masks + continue + mask_image[(selected_masks[index]==1).cpu().numpy()] = mask_id + mask_id += 1 + cv2.imwrite(os.path.join(output_dir, os.path.basename(path).split('.')[0] + '.png'), mask_image) \ No newline at end of file diff --git a/MaskClustering/prep_arkit.py b/MaskClustering/prep_arkit.py new file mode 100644 index 0000000000000000000000000000000000000000..25fdd8a4d0deebd7783748f6d3133798c972fa28 --- /dev/null +++ b/MaskClustering/prep_arkit.py @@ -0,0 +1,117 @@ +from pathlib import Path +import shutil +import os +import json +import numpy as np +import pandas as pd +import matplotlib.pyplot as plt +import torch +import pickle +import cv2 +from tqdm.contrib.concurrent import process_map + + + +dust3r_path = Path("/home/jovyan/users/bulat/workspace/3drec/Indoor/DUSt3R/res/scannet_posed_35") +pcds_path = Path("/home/jovyan/users/bulat/workspace/3drec/Indoor/DUSt3R/res_scennet_posed_35_new") + +def resize_intrinsics(K, base_shape, new_shape): + fx = K[0, 0] + fy = K[1, 1] + cx = K[0, 2] + cy = K[1, 2] + + fx_new = fx * new_shape[0] / base_shape[0] + fy_new = fy * new_shape[1] / base_shape[1] + cx_new = cx * new_shape[0] / base_shape[0] + cy_new = cy * new_shape[1] / base_shape[1] + + return np.array([[fx_new, 0, cx_new, 0], [0, fy_new, cy_new, 0], [0, 0, 1, 0], [0, 0, 0, 1]]) + +def prepare_scene(scene_id): + + scannet_processed_path = Path("/home/jovyan/users/bulat/workspace/3drec/Indoor/MaskClustering/data/scannet/processed") / scene_id + out_path = Path("data/scannet_dust3r_posed_35_bulat/processed") / scene_id + out_path.mkdir(parents=True, exist_ok=True) + + path_to_scene = dust3r_path / scene_id + scene_params = torch.load(path_to_scene / 'scene_params.pt') + pcd_path = pcds_path / (scene_id + '.ply') + + + poses = scene_params['poses'] + depths = scene_params['depths'] + Ks = scene_params['Ks'] + image_names = [Path(image_name) for image_name in scene_params['image_files']] + shutil.copy(pcd_path, out_path / f'{scene_id}_vh_clean_2.ply') + + + path_to_out_intrinsic = out_path / 'intrinsic' + path_to_out_intrinsic.mkdir(parents=True, exist_ok=True) + + shutil.copy(scannet_processed_path / 'intrinsic' / 'intrinsic_color.txt', path_to_out_intrinsic / 'intrinsic_color.txt') + shutil.copy(scannet_processed_path / 'intrinsic' / 'intrinsic_depth.txt', path_to_out_intrinsic / 'intrinsic_depth.txt') + shutil.copy(scannet_processed_path / 'intrinsic' / 'extrinsic_color.txt', path_to_out_intrinsic / 'extrinsic_color.txt') + shutil.copy(scannet_processed_path / 'intrinsic' / 'extrinsic_depth.txt', path_to_out_intrinsic / 'extrinsic_depth.txt') + + # base_intrinsic_path = f'/home/jovyan/users/lemeshko/mmdetection3d/data/scannet/posed_images/{scene_id}/intrinsic.txt' + # base_intrinsic = np.loadtxt(base_intrinsic_path) + + # np.savetxt(path_to_out_intrinsic / 'intrinsic_color.txt', resize_intrinsics(Ks.mean(0), (512, 384), (1280, 960))) + # np.savetxt(path_to_out_intrinsic / 'intrinsic_depth.txt', resize_intrinsics(Ks.mean(0), (512, 384), (640, 480))) + # np.savetxt(path_to_out_intrinsic / 'extrinsic_color.txt', np.eye(4)) + # np.savetxt(path_to_out_intrinsic / 'extrinsic_depth.txt', np.eye(4)) + + path_to_out_color = out_path / 'color' + path_to_out_color.mkdir(parents=True, exist_ok=True) + + + path_to_out_depth = out_path / 'depth' + path_to_out_depth.mkdir(parents=True, exist_ok=True) + + path_to_out_pose = out_path / 'pose' + path_to_out_pose.mkdir(parents=True, exist_ok=True) + + + image_file = Path(image_names[0]) + gt_depth = image_file.parent.parent / 'depth_maps' / (image_file.stem + ".png") + gt_pose = image_file.parent.parent / 'poses' / (image_file.stem + ".txt") + + print(gt_depth) + # gt_depth = cv2.imread(str(gt_depth), -1) / 1000.0 + # gt_pose = np.loadtxt(str(gt_pose)) + + + # h, w = gt_depth.shape + # depth = depths[0].numpy().astype(np.float32) + # depth = cv2.resize(depth, (w, h), interpolation=cv2.INTER_NEAREST) + + # valid_mask = (gt_depth > 1e-10) & (depth > 1e-10) + # depth = depth[valid_mask] + # gt_depth = gt_depth[valid_mask] + + # scale = np.median(gt_depth / depth) + # first_pose = np.linalg.inv(poses[0].numpy()) + + + for i, image_name in enumerate(image_names): + image_name = Path(image_name) + def get_image_name(image_name): + return str(int(image_name.stem)) + shutil.copy(Path("/home/jovyan/users/lemeshko/mmdetection3d/data/scannet/posed_images") / scene_id / image_name.name, path_to_out_color / (get_image_name(image_name) + '.jpg')) + + depth_map = depths[i].numpy() * 1000 + depth_map = cv2.resize(depth_map, (640, 480)) + depth_map = depth_map.astype(np.uint16) + cv2.imwrite(str(path_to_out_depth / (get_image_name(image_name) + '.png')), depth_map) + + pose = poses[i].numpy() + np.savetxt(str(path_to_out_pose / (get_image_name(image_name) + '.txt')), pose) + + +if __name__ == "__main__": + pkl_path = '/home/jovyan/users/lemeshko/mmdetection3d/data/scannetOV20/scannet_oneformer3d_infos_val.pkl' + with open(pkl_path, 'rb') as file: + lst = pickle.load(file)['data_list'] + scenes = sorted([val['lidar_points']['lidar_path'][:-4] for val in lst]) + process_map(prepare_scene, scenes, chunksize=10) \ No newline at end of file diff --git a/MaskClustering/prep_scannet.py b/MaskClustering/prep_scannet.py new file mode 100644 index 0000000000000000000000000000000000000000..25fdd8a4d0deebd7783748f6d3133798c972fa28 --- /dev/null +++ b/MaskClustering/prep_scannet.py @@ -0,0 +1,117 @@ +from pathlib import Path +import shutil +import os +import json +import numpy as np +import pandas as pd +import matplotlib.pyplot as plt +import torch +import pickle +import cv2 +from tqdm.contrib.concurrent import process_map + + + +dust3r_path = Path("/home/jovyan/users/bulat/workspace/3drec/Indoor/DUSt3R/res/scannet_posed_35") +pcds_path = Path("/home/jovyan/users/bulat/workspace/3drec/Indoor/DUSt3R/res_scennet_posed_35_new") + +def resize_intrinsics(K, base_shape, new_shape): + fx = K[0, 0] + fy = K[1, 1] + cx = K[0, 2] + cy = K[1, 2] + + fx_new = fx * new_shape[0] / base_shape[0] + fy_new = fy * new_shape[1] / base_shape[1] + cx_new = cx * new_shape[0] / base_shape[0] + cy_new = cy * new_shape[1] / base_shape[1] + + return np.array([[fx_new, 0, cx_new, 0], [0, fy_new, cy_new, 0], [0, 0, 1, 0], [0, 0, 0, 1]]) + +def prepare_scene(scene_id): + + scannet_processed_path = Path("/home/jovyan/users/bulat/workspace/3drec/Indoor/MaskClustering/data/scannet/processed") / scene_id + out_path = Path("data/scannet_dust3r_posed_35_bulat/processed") / scene_id + out_path.mkdir(parents=True, exist_ok=True) + + path_to_scene = dust3r_path / scene_id + scene_params = torch.load(path_to_scene / 'scene_params.pt') + pcd_path = pcds_path / (scene_id + '.ply') + + + poses = scene_params['poses'] + depths = scene_params['depths'] + Ks = scene_params['Ks'] + image_names = [Path(image_name) for image_name in scene_params['image_files']] + shutil.copy(pcd_path, out_path / f'{scene_id}_vh_clean_2.ply') + + + path_to_out_intrinsic = out_path / 'intrinsic' + path_to_out_intrinsic.mkdir(parents=True, exist_ok=True) + + shutil.copy(scannet_processed_path / 'intrinsic' / 'intrinsic_color.txt', path_to_out_intrinsic / 'intrinsic_color.txt') + shutil.copy(scannet_processed_path / 'intrinsic' / 'intrinsic_depth.txt', path_to_out_intrinsic / 'intrinsic_depth.txt') + shutil.copy(scannet_processed_path / 'intrinsic' / 'extrinsic_color.txt', path_to_out_intrinsic / 'extrinsic_color.txt') + shutil.copy(scannet_processed_path / 'intrinsic' / 'extrinsic_depth.txt', path_to_out_intrinsic / 'extrinsic_depth.txt') + + # base_intrinsic_path = f'/home/jovyan/users/lemeshko/mmdetection3d/data/scannet/posed_images/{scene_id}/intrinsic.txt' + # base_intrinsic = np.loadtxt(base_intrinsic_path) + + # np.savetxt(path_to_out_intrinsic / 'intrinsic_color.txt', resize_intrinsics(Ks.mean(0), (512, 384), (1280, 960))) + # np.savetxt(path_to_out_intrinsic / 'intrinsic_depth.txt', resize_intrinsics(Ks.mean(0), (512, 384), (640, 480))) + # np.savetxt(path_to_out_intrinsic / 'extrinsic_color.txt', np.eye(4)) + # np.savetxt(path_to_out_intrinsic / 'extrinsic_depth.txt', np.eye(4)) + + path_to_out_color = out_path / 'color' + path_to_out_color.mkdir(parents=True, exist_ok=True) + + + path_to_out_depth = out_path / 'depth' + path_to_out_depth.mkdir(parents=True, exist_ok=True) + + path_to_out_pose = out_path / 'pose' + path_to_out_pose.mkdir(parents=True, exist_ok=True) + + + image_file = Path(image_names[0]) + gt_depth = image_file.parent.parent / 'depth_maps' / (image_file.stem + ".png") + gt_pose = image_file.parent.parent / 'poses' / (image_file.stem + ".txt") + + print(gt_depth) + # gt_depth = cv2.imread(str(gt_depth), -1) / 1000.0 + # gt_pose = np.loadtxt(str(gt_pose)) + + + # h, w = gt_depth.shape + # depth = depths[0].numpy().astype(np.float32) + # depth = cv2.resize(depth, (w, h), interpolation=cv2.INTER_NEAREST) + + # valid_mask = (gt_depth > 1e-10) & (depth > 1e-10) + # depth = depth[valid_mask] + # gt_depth = gt_depth[valid_mask] + + # scale = np.median(gt_depth / depth) + # first_pose = np.linalg.inv(poses[0].numpy()) + + + for i, image_name in enumerate(image_names): + image_name = Path(image_name) + def get_image_name(image_name): + return str(int(image_name.stem)) + shutil.copy(Path("/home/jovyan/users/lemeshko/mmdetection3d/data/scannet/posed_images") / scene_id / image_name.name, path_to_out_color / (get_image_name(image_name) + '.jpg')) + + depth_map = depths[i].numpy() * 1000 + depth_map = cv2.resize(depth_map, (640, 480)) + depth_map = depth_map.astype(np.uint16) + cv2.imwrite(str(path_to_out_depth / (get_image_name(image_name) + '.png')), depth_map) + + pose = poses[i].numpy() + np.savetxt(str(path_to_out_pose / (get_image_name(image_name) + '.txt')), pose) + + +if __name__ == "__main__": + pkl_path = '/home/jovyan/users/lemeshko/mmdetection3d/data/scannetOV20/scannet_oneformer3d_infos_val.pkl' + with open(pkl_path, 'rb') as file: + lst = pickle.load(file)['data_list'] + scenes = sorted([val['lidar_points']['lidar_path'][:-4] for val in lst]) + process_map(prepare_scene, scenes, chunksize=10) \ No newline at end of file diff --git a/MaskClustering/preprocess/matterport3d/category_mapping.tsv b/MaskClustering/preprocess/matterport3d/category_mapping.tsv new file mode 100755 index 0000000000000000000000000000000000000000..b79422c439a7db72788a9e059a53d0a63a856a6d --- /dev/null +++ b/MaskClustering/preprocess/matterport3d/category_mapping.tsv @@ -0,0 +1,1660 @@ +index raw_category category count nyuId nyu40id eigen13id nyuClass nyu40class eigen13class ModelNet40 ModelNet10 ShapeNetCore55 synsetoffset wnsynsetid wnsynsetkey mpcat40index mpcat40 +1 wall wall 7667 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +2 door door 2544 28 8 12 door door Wall door n03221720 door.n.01 4 door +3 ceiling ceiling 2363 4 22 3 ceiling ceiling Ceiling n02990373 ceiling.n.01 17 ceiling +4 floor floor 2252 11 2 5 floor floor Floor n03365592 floor.n.01 2 floor +5 picture picture 2125 64 11 8 picture picture Picture n03931044 picture.n.01 6 picture +6 window window 2013 59 9 13 window window Window n04587648 window.n.01 9 window +7 chair chair 1947 5 5 4 chair chair Chair chair chair chair 03001627 n03001627 chair.n.01 3 chair +8 doorframe door frame 1935 28 8 12 door door Wall door n03221720 door.n.01 4 door +9 remove remove 1622 0 0 0 void void void 0 void +10 pillow pillow 1335 119 18 7 pillow pillow Objects pillow 03938244 n03938244 pillow.n.01 8 cushion +11 object object 1145 40 7 otherprop Objects n00002684 object.n.01 39 objects +12 light light 967 144 35 7 lamp lamp Objects lamp lamp 03636649 n03636649 lamp.n.02 28 lighting +13 cabinet cabinet 785 3 3 6 cabinet cabinet Furniture cabinet 02933112 n02933112 cabinet.n.01 7 cabinet +14 curtain curtain 692 89 16 13 curtain curtain Window curtain n03151077 curtain.n.01 12 curtain +15 table table 691 19 7 10 table table Table table table table 04379243 n04379243 table.n.02 5 table +16 plant plant 632 82 40 7 plant otherprop Objects plant n00017222 plant.n.02 14 plant +17 decoration decoration 532 40 7 otherprop Objects n03169390 decoration.n.01 39 objects +18 window frame window frame 526 59 9 13 window window Window n04589593 window_frame.n.01 9 window +19 lamp lamp 540 144 35 7 lamp lamp Objects lamp lamp 03636649 n03636649 lamp.n.02 28 lighting +20 mirror mirror 488 122 19 7 mirror mirror Objects n03773035 mirror.n.01 21 mirror +21 towel towel 437 135 27 7 towel towel Objects n04459362 towel.n.01 20 towel +22 sink sink 417 24 34 7 sink sink Objects sink n04223580 sink.n.01 15 sink +23 shelf shelf 417 42 15 6 shelves shelves Furniture n04190052 shelf.n.01 31 shelving +24 couch couch 290 83 6 9 sofa sofa Sofa sofa sofa sofa 04256520 n04256520 sofa.n.01 10 sofa +25 dining chair dining chair 277 5 5 4 chair chair Chair chair chair chair 03001627 n03001627 chair.n.01 3 chair +26 bed bed 270 157 4 1 bed bed Bed bed bed bed 02818832 n02818832 bed.n.01 11 bed +27 nightstand nightstand 275 158 32 6 night stand night stand Furniture night_stand night_stand n03015254 chest_of_drawers.n.01 13 chest_of_drawers +28 toilet toilet 268 124 33 7 toilet toilet Objects toilet toilet n04446276 toilet.n.01 18 toilet +29 sofa chair sofa chair 262 5 5 4 chair chair Chair chair chair chair 03001627 n03001627 chair.n.01 3 chair +30 column pillar 243 94 38 7 column otherstructure Objects n03073977 column.n.07 24 column +31 handrail handrail 241 453 38 7 banister otherstructure Objects n02788148 bannister.n.02 30 railing +32 stair stair 241 215 38 7 stairs otherstructure Objects stairs n04314914 step.n.04 16 stairs +33 stool stool 232 150 40 7 stool otherprop Objects stool n04326896 stool.n.01 19 stool +34 armchair armchair 224 5 5 4 chair chair Chair chair chair chair 03001627 n02738535 armchair.n.01 3 chair +35 kitchen cabinet kitchen cabinet 212 3 3 6 cabinet cabinet Furniture n02933112 cabinet.n.01 7 cabinet +36 vase vase 205 78 40 7 vase otherprop Objects vase jar 03593526 n04522168 vase.n.01 39 objects +37 cushion cushion 204 119 18 7 pillow pillow Objects n03151500 cushion.n.03 8 cushion +38 tv tv 204 172 25 11 television television TV tv or monitor 03211117 n03211117 display.n.06 22 tv_monitor +39 door frame door frame 190 28 8 12 door door Wall door n03221720 door.n.01 4 door +40 unknown unknown 186 20 40 7 unknown otherprop Objects n08632096 unknown.n.01 41 unlabeled +41 pot pot 185 16 40 7 pot otherprop Objects n03991062 pot.n.04 39 objects +42 desk desk 187 36 14 10 desk desk Table desk desk table 04379243 n03179701 desk.n.01 5 table +43 painting picture 194 64 11 8 picture picture Picture n03931044 picture.n.01 6 picture +44 roof roof 173 4 22 3 ceiling ceiling Ceiling n04105068 roof.n.01 17 ceiling +45 box box 172 26 29 7 box box Objects n02883344 box.n.01 39 objects +46 shower wall shower wall 168 21 1 12 wall wall Wall n04208936 shower.n.01 23 shower +47 coffee table coffee table 159 356 39 6 coffee table otherfurniture Furniture table table table 04379243 n03063968 coffee_table.n.01 5 table +48 countertop countertop 159 7 12 6 counter counter Furniture n03118245 countertop.n.01 26 counter +49 bench bench 154 204 39 6 bench otherfurniture Furniture bench bench 02828884 n02828884 bench.n.01 34 seating +50 wall frame picture 142 64 11 8 picture picture Picture n03931044 picture.n.01 6 picture +51 trash can trashcan 131 12 39 6 garbage bin otherfurniture Furniture trash_bin 02747177 n02747177 ashcan.n.01 39 objects +52 fireplace fireplace 128 372 38 7 fireplace otherstructure Objects n03346455 fireplace.n.01 27 fireplace +53 clothes clothes 125 141 21 7 clothes clothes Objects n02728440 apparel.n.01 38 clothes +54 pillar pillar 117 94 38 7 column otherstructure Objects n03073977 column.n.07 24 column +55 bathtub bathtub 121 136 36 7 bathtub bathtub Objects bathtub bathtub tub 02808440 n02808440 bathtub.n.01 25 bathtub +56 ceiling duct ceiling duct 110 4 22 3 ceiling ceiling Ceiling n02990373 ceiling.n.01 17 ceiling +57 bath cabinet bath cabinet 111 3 3 6 cabinet cabinet Furniture cabinet 02933112 n02933112 cabinet.n.01 7 cabinet +58 book book 104 1 23 2 book books Books n02870526 book.n.11 39 objects +59 beam beam 101 38 7 otherstructure Objects n02815950 beam.n.02 29 beam +60 vent vent 101 25 38 7 air vent otherstructure Objects n04526241 vent.n.01 40 misc +61 shower floor shower floor 97 11 2 5 floor floor Floor n04208936 shower.n.01 23 shower +62 faucet faucet 97 9 40 7 faucet otherprop Objects faucet 03325088 n03325088 faucet.n.01 39 objects +63 photo photo 100 508 40 7 photo otherprop Objects n03925226 photograph.n.01 6 picture +64 delete remove 90 0 0 0 void void void 0 void +65 toilet paper toilet paper 90 139 40 7 toilet paper otherprop Objects n15075141 toilet_tissue.n.01 39 objects +66 counter counter 91 7 12 6 counter counter Furniture table table table 04379243 n03116530 counter.n.01 26 counter +67 fan fan 85 74 40 7 fan otherprop Objects n03320046 fan.n.01 39 objects +68 step step 86 38 7 otherstructure Objects n04314914 step.n.04 16 stairs +69 table lamp table lamp 96 144 35 7 lamp lamp Objects lamp lamp 03636649 n04380533 table_lamp.n.01 28 lighting +70 wall / other room wall /otherroom 84 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +71 washbasin washbasin 91 24 34 7 sink sink Objects sink n04553920 washbasin.n.01 15 sink +72 rail railing 83 497 38 7 railing otherstructure Objects n04047401 railing.n.01 30 railing +73 side table table 85 19 7 10 table table Table table table table 04379243 n04379243 table.n.02 5 table +74 decor decoration 79 40 7 otherprop Objects n03169390 decoration.n.01 39 objects +75 shelves shelving 79 42 15 6 shelves shelves Furniture n04190052 shelf.n.01 31 shelving +76 statue statue 78 294 40 7 sculpture otherprop Objects n04306847 statue.n.01 39 objects +77 dresser dresser 79 169 17 6 dresser dresser Furniture dresser dresser n03015254 chest_of_drawers.n.01 13 chest_of_drawers +78 stairs stair 76 215 38 7 stairs otherstructure Objects stairs n04314914 step.n.04 16 stairs +79 rug rug 77 130 40 7 rug floor mat Objects n04118021 rug.n.01 2 floor +80 ottoman ottoman 79 359 39 6 ottoman otherfurniture Furniture stool n03380724 footstool.n.01 19 stool +81 round table table 74 19 7 10 table table Table table table table 04379243 n04379243 table.n.02 5 table +82 bottle bottle 101 2 40 7 bottle otherprop Objects bottle bottle 02876657 n02876657 bottle.n.01 39 objects +83 kitchen counter kitchen counter 73 7 12 6 counter counter Furniture table table table 04379243 n03116530 counter.n.01 26 counter +84 windowframe window frame 66 59 9 13 window window Window n04589593 window_frame.n.01 9 window +85 office chair office chair 70 5 5 4 chair chair Chair chair chair chair 03001627 n04373704 swivel_chair.n.01 3 chair +86 frame frame 66 38 7 otherstructure Objects 40 misc +87 refrigerator refrigerator 65 17 24 6 refridgerator refridgerator Furniture n04070727 refrigerator.n.01 37 appliances +88 bookshelf bookshelf 63 88 10 6 bookshelf bookshelf Furniture bookshelf bookshelf 02871439 n02871439 bookshelf.n.01 31 shelving +89 end table end table 67 19 7 10 table table Table table table table 04379243 n04379243 table.n.02 5 table +90 wardrobe wardrobe 64 772 39 6 wardrobe otherfurniture Furniture wardrobe n04550184 wardrobe.n.01 36 furniture +91 toiletry toiletry 63 40 7 otherprop Objects n04447443 toiletry.n.01 39 objects +92 windowsill window frame 62 59 9 13 window window Window n04589593 window_frame.n.01 9 window +93 pipe pipe 62 41 40 7 pipe otherprop Objects n03944672 pipe.n.02 40 misc +94 monitor monitor 60 49 40 7 monitor otherprop Objects monitor monitor tv or monitor 03211117 n03782190 monitor.n.04 22 tv_monitor +95 stand stand 62 50 39 6 stand otherfurniture Furniture table table table 04379243 n04301000 stand.n.04 5 table +96 drawers drawer 60 174 39 6 drawer otherfurniture Furniture n03233905 drawer.n.01 13 chest_of_drawers +97 dining table dining table 61 19 7 10 table table Table table table table 04379243 n04379243 table.n.02 5 table +98 container container 59 140 40 7 container otherprop Objects n03094503 container.n.01 39 objects +99 light switch light switch 59 301 38 7 light switch otherstructure Objects n04372370 switch.n.01 39 objects +100 skylight skylight 57 59 9 13 window window Window n04232800 skylight.n.01 9 window +101 purse purse 57 181 40 7 purse otherprop Objects n02774152 bag.n.04 39 objects +102 wall /otherroom wall /otherroom 55 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +103 sofa couch 56 83 6 9 sofa sofa Sofa sofa sofa sofa 04256520 n04256520 sofa.n.01 10 sofa +104 books book 53 1 23 2 book books Books n02870526 book.n.11 39 objects +105 doorway doorway 54 609 38 7 door way otherstructure Objects door n03224032 doorway.n.01 4 door +106 railing railing 52 497 38 7 railing otherstructure Objects n04047401 railing.n.01 30 railing +107 wainscotting paneling 52 21 1 12 wall wall Wall n03882611 paneling.n.01 1 wall +108 basket basket 52 39 40 7 basket otherprop Objects basket 02801938 n02801938 basket.n.01 39 objects +109 closet closet 50 40 7 otherprop Objects n03148324 cupboard.n.01 7 cabinet +110 arch arch 48 40 7 otherprop Objects n02733524 arch.n.04 40 misc +111 chandelier chandelier 48 342 38 7 chandelier otherstructure Objects n03005285 chandelier.n.01 28 lighting +112 oven oven 47 238 38 7 oven otherstructure Objects n03862676 oven.n.01 37 appliances +113 clock clock 47 56 40 7 clock otherprop Objects clock 03046257 n03046257 clock.n.01 39 objects +114 footstool footstool 48 359 39 6 ottoman otherfurniture Furniture stool n03380724 footstool.n.01 19 stool +115 stove stove 44 242 38 7 stove otherstructure Objects stove 04330267 n04330267 stove.n.02 37 appliances +116 trashcan trashcan 44 12 39 6 garbage bin otherfurniture Furniture trash_bin 02747177 n02747177 ashcan.n.01 39 objects +117 drawer drawer 45 174 39 6 drawer otherfurniture Furniture n03233905 drawer.n.01 13 chest_of_drawers +118 bathroom countertop object object 43 40 7 otherprop Objects n00002684 object.n.01 39 objects +119 washing machine washing machine 43 278 39 6 washing machine otherfurniture Furniture washing_machine 04554684 n04554684 washer.n.03 37 appliances +120 shower curtain shower curtain 43 123 28 7 shower curtain shower curtain Objects curtain n04209239 shower_curtain.n.01 12 curtain +121 rack rack 41 50 39 6 stand otherfurniture Furniture n04038440 rack.n.05 31 shelving +122 art picture 40 64 11 8 picture picture Picture n03931044 picture.n.01 6 picture +123 firealarm fire alarm 37 338 40 7 fire alarm otherprop Objects n03343737 fire_alarm.n.02 39 objects +124 bin bin 37 307 40 7 bin otherprop Objects n02839910 bin.n.01 39 objects +125 chest chest 36 344 39 6 chest otherfurniture Furniture dresser dresser 40 misc +126 microwave microwave 36 13 40 7 microwave otherprop Objects microwave 03761084 n03761084 microwave.n.02 37 appliances +127 blinds blinds 36 80 13 13 blinds blinds Window n04589190 window_blind.n.01 32 blinds +128 bowl bowl 35 22 40 7 bowl otherprop Objects bowl bowl 02880940 n02880940 bowl.n.03 39 objects +129 ceiling pipe ceiling pipe 34 4 22 3 ceiling ceiling Ceiling n02990373 ceiling.n.01 17 ceiling +130 tree tree 34 82 40 7 plant otherprop Objects plant n13104059 tree.n.01 14 plant +131 vanity vanity 34 169 17 6 dresser dresser Furniture dresser dresser table 04379243 n03238586 dressing_table.n.01 5 table +132 ceiling fan ceiling fan 37 74 40 7 fan otherprop Objects n03320046 fan.n.01 39 objects +133 tissue box tissue box 34 138 40 7 tissue box otherprop Objects n02883344 box.n.01 39 objects +134 desk chair desk chair 34 5 5 4 chair chair Chair chair chair chair 03001627 n03001627 chair.n.01 3 chair +135 plate plate 32 233 40 7 plate otherprop Objects n03959485 plate.n.04 39 objects +136 tv stand tv stand 32 291 39 6 tv stand otherfurniture Furniture tv_stand n03290653 entertainment_center.n.01 36 furniture +137 shoes shoe 32 149 40 7 shoe otherprop Objects n04199027 shoe.n.01 39 objects +138 heater heater 32 111 39 6 heater otherfurniture Furniture n03508101 heater.n.01 39 objects +139 bedframe bedframe 33 157 4 1 bed bed Bed n02822579 bedstead.n.01 11 bed +140 headboard headboard 33 161 39 6 headboard otherfurniture Furniture n03502200 headboard.n.01 11 bed +141 post post 32 94 38 7 column otherstructure Objects n03988170 post.n.04 24 column +142 swivel chair swivel chair 31 5 5 4 chair chair Chair chair chair chair 03001627 n04373704 swivel_chair.n.01 3 chair +143 pedestal pedestal 31 50 39 6 stand otherfurniture Furniture 40 misc +144 fence fence 31 38 7 otherstructure Objects n03327234 fence.n.01 40 misc +145 ceiling pipes ceiling pipe 30 4 22 3 ceiling ceiling Ceiling n02990373 ceiling.n.01 17 ceiling +146 pew pew 28 204 39 6 bench otherfurniture Furniture bench bench 02828884 n03920867 pew.n.01 34 seating +147 decorative object decoration 28 40 7 otherprop Objects n03169390 decoration.n.01 39 objects +148 bucket bucket 28 427 40 7 bucket otherprop Objects n02909870 bucket.n.01 39 objects +149 mask decorative mask 27 40 7 otherprop Objects n03169390 decoration.n.01 39 objects +150 candle candle 27 137 40 7 candle otherprop Objects lamp n02948072 candle.n.01 39 objects +151 flowerpot flowerpot 28 146 40 7 flower pot otherprop Objects flower_pot flower pot 03991062 n03991062 pot.n.04 39 objects +152 speaker speaker 27 54 40 7 speaker otherprop Objects speaker 03691459 n03691459 loudspeaker.n.01 39 objects +153 seat seat 26 524 39 6 furniture otherfurniture Furniture n04161981 seat.n.03 34 seating +154 sign sign 25 208 40 7 sign otherprop Objects n04217882 signboard.n.01 40 misc +155 air conditioner air conditioner 26 79 38 7 air conditioner otherstructure Objects n02686379 air_conditioner.n.01 39 objects +156 shower curtain rod shower curtain rod 25 40 7 otherprop Objects n04100174 rod.n.01 12 curtain +157 unknown / other room unknown /otherroom 24 20 40 7 unknown otherprop Objects n08632096 unknown.n.01 41 unlabeled +158 flowers plant 25 82 40 7 plant otherprop Objects plant n00017222 plant.n.02 14 plant +159 clutter clutter 24 40 7 otherprop Objects 40 misc +160 pillows pillow 24 119 18 7 pillow pillow Objects pillow 03938244 n03938244 pillow.n.01 8 cushion +161 plants plant 24 82 40 7 plant otherprop Objects plant n00017222 plant.n.02 14 plant +162 wall \other room wall /otherroom 24 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +163 fire extinguisher fire extinguisher 24 10 40 7 fire extinguisher otherprop Objects n03345837 fire_extinguisher.n.01 39 objects +164 towels towel 24 135 27 7 towel towel Objects n04459362 towel.n.01 20 towel +165 curtains curtain 23 89 16 13 curtain curtain Window curtain n03151077 curtain.n.01 12 curtain +166 curtain rod curtain rod 23 582 38 7 curtain rod otherstructure Objects n04100174 rod.n.01 12 curtain +167 kitchen countertop object object 23 40 7 otherprop Objects n00002684 object.n.01 39 objects +168 mat mat 23 143 20 5 floor mat floor mat Floor n03727837 mat.n.01 2 floor +169 flower plant 23 82 40 7 plant otherprop Objects plant n00017222 plant.n.02 14 plant +170 sculpture sculpture 23 294 40 7 sculpture otherprop Objects n04157320 sculpture.n.01 39 objects +171 shelving shelving 22 42 15 6 shelves shelves Furniture n04190052 shelf.n.01 31 shelving +172 wall/other room wall /otherroom 22 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +173 knickknack object 21 40 7 otherprop Objects n00002684 object.n.01 39 objects +174 printer printer 21 66 40 7 printer otherprop Objects printer 04004475 n04004475 printer.n.03 39 objects +175 wall behind wall /otherroom 21 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +176 telephone telephone 21 32 40 7 telephone otherprop Objects telephone 04401088 n04401088 telephone.n.01 39 objects +177 bedside table nightstand 21 158 32 6 night stand night stand Furniture night_stand night_stand n03015254 chest_of_drawers.n.01 13 chest_of_drawers +178 moulding molding 21 38 7 otherstructure Objects n02800354 baseboard.n.01 1 wall +179 handbag handbag 21 40 7 otherprop Objects n02774152 bag.n.04 39 objects +180 wall /other room wall /otherroom 21 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +181 blanket blanket 21 312 40 7 blanket otherprop Objects n02849154 blanket.n.01 39 objects +182 shower shower 21 38 7 otherstructure Objects n04208936 shower.n.01 23 shower +183 steps step 20 38 7 otherstructure Objects n04314914 step.n.04 16 stairs +184 switch switch 21 40 7 otherprop Objects n04372370 switch.n.01 39 objects +185 toilet paper dispenser toilet paper dispenser 20 40 7 otherprop Objects 40 misc +186 objects object 21 40 7 otherprop Objects n00002684 object.n.01 39 objects +187 handle handle 20 758 40 7 handle otherprop Objects n03485997 handle.n.01 39 objects +188 frame /outside frame /outside 20 38 7 otherstructure Objects 40 misc +189 screen screen 20 89 16 13 curtain curtain Window curtain n03151077 curtain.n.01 12 curtain +190 shower head showerhead 19 650 40 7 shower head otherprop Objects n04209383 showerhead.n.01 23 shower +191 baracade barricade 19 40 7 otherprop Objects n04096848 roadblock.n.02 40 misc +192 picture frame picture frame 25 64 11 8 picture picture Picture n03931765 picture_frame.n.01 6 picture +193 soap soap 19 133 40 7 soap otherprop Objects n04253437 soap.n.01 39 objects +194 staircase railing banister 18 453 38 7 banister otherstructure Objects n02788148 bannister.n.02 30 railing +195 keyboard keyboard 18 47 40 7 keyboard otherprop Objects keyboard computer keyboard 03085013 n03085013 computer_keyboard.n.01 39 objects +196 thermostat thermostat 18 110 40 7 thermostat otherprop Objects n04422875 thermostat.n.01 39 objects +197 radiator radiator 18 236 39 6 radiator otherfurniture Furniture n04041069 radiator.n.02 39 objects +198 kitchen island kitchen island 18 456 38 7 kitchen island otherstructure Objects n03620600 kitchen_island.n.01 26 counter +199 paper towel paper towel 18 113 40 7 paper towel otherprop Objects n03887697 paper_towel.n.01 20 towel +200 wall decoration picture 17 64 11 8 picture picture Picture n03931044 picture.n.01 6 picture +201 phone telephone 17 32 40 7 telephone otherprop Objects telephone 04401088 n04401088 telephone.n.01 39 objects +202 mirror frame mirror 17 122 19 7 mirror mirror Objects n03773035 mirror.n.01 21 mirror +203 clothes dryer clothes dryer 18 39 6 otherfurniture Furniture n03251766 dryer.n.01 37 appliances +204 panel panel 17 559 40 7 sheet otherprop Objects n03882058 panel.n.01 35 board_panel +205 glass glass 16 612 38 7 glass otherstructure Objects n03438257 glass.n.02 39 objects +206 soap dispenser soap dispenser 16 40 7 otherprop Objects n04254120 soap_dispenser.n.01 39 objects +207 dishwasher dishwasher 16 8 38 7 dishwasher otherstructure Objects dishwasher 03207941 n03207941 dishwasher.n.01 37 appliances +208 cup cup 16 35 40 7 cup otherprop Objects cup cup or mug 03797390 n03797390 mug.n.04 39 objects +209 bathroom cabinet bathroom cabinet 17 3 3 6 cabinet cabinet Furniture cabinet 02933112 n02933112 cabinet.n.01 7 cabinet +210 ladder ladder 17 48 39 6 ladder otherfurniture Furniture stairs n03632277 ladder.n.01 16 stairs +211 garage door garage door 16 850 38 7 garage door otherstructure Objects door 4 door +212 hat hat 15 193 40 7 hat otherprop Objects n03497657 hat.n.01 38 clothes +213 chest of drawers chest of drawers 15 524 39 6 furniture otherfurniture Furniture dresser dresser n03015254 chest_of_drawers.n.01 13 chest_of_drawers +214 exit sign exit sign 15 86 40 7 exit sign otherprop Objects 40 misc +215 sidetable side table 15 19 7 10 table table Table table table table 04379243 n04379243 table.n.02 5 table +216 office table office table 15 19 7 10 table table Table table table table 04379243 n04379243 table.n.02 5 table +217 piano piano 15 298 39 6 piano otherfurniture Furniture piano piano 03928116 n03928116 piano.n.01 39 objects +218 painter picture 17 64 11 8 picture picture Picture n03931044 picture.n.01 6 picture +219 board board 15 408 38 7 board otherstructure Objects 35 board_panel +220 windows window 14 59 9 13 window window Window n04587648 window.n.01 9 window +221 archway archway 14 21 1 12 wall wall Wall n02734217 arch.n.03 4 door +222 rope rope 14 560 40 7 rope otherprop Objects n04108268 rope.n.01 39 objects +223 ball ball 15 60 40 7 ball otherprop Objects 40 misc +224 gym equipment gym equipment 14 457 39 6 excercise equipment otherfurniture Furniture n04285146 sports_equipment.n.01 33 gym_equipment +225 clothes hangers clothes hanger 13 211 40 7 hanger otherprop Objects n03057920 coat_hanger.n.01 39 objects +226 bathroom object object 13 40 7 otherprop Objects n00002684 object.n.01 39 objects +227 easy chair easy chair 13 5 5 4 chair chair Chair chair chair chair 03001627 n03262932 easy_chair.n.01 3 chair +228 lounge chair lounge chair 15 5 5 4 chair chair Chair chair chair chair 03001627 n03262932 easy_chair.n.01 3 chair +229 furniture furniture 13 524 39 6 furniture otherfurniture Furniture n03405725 furniture.n.01 36 furniture +230 cabinets cabinet 16 3 3 6 cabinet cabinet Furniture cabinet 02933112 n02933112 cabinet.n.01 7 cabinet +231 carpet carpet 14 130 40 7 rug floor mat Objects n04118021 rug.n.01 2 floor +232 food food 13 40 7 otherprop Objects n00021265 food.n.01 40 misc +233 plant pot pot 13 16 40 7 pot otherprop Objects n03991062 pot.n.04 39 objects +234 duct duct 13 40 7 otherprop Objects n03253398 duct.n.03 40 misc +235 ridge ridge 13 40 7 otherprop Objects 40 misc +236 candlestick candlestick 12 148 40 7 candlestick otherprop Objects n02948557 candlestick.n.01 39 objects +237 computer desk computer desk 12 36 14 10 desk desk Table desk desk table 04379243 n03179701 desk.n.01 5 table +238 shower door shower door 12 28 8 12 door door Wall door n04208936 shower.n.01 23 shower +239 trash trashcan 12 12 39 6 garbage bin otherfurniture Furniture trash_bin 02747177 n02747177 ashcan.n.01 39 objects +240 crown molding molding 12 38 7 otherstructure Objects n02800354 baseboard.n.01 1 wall +241 wall sconce sconce 12 62 38 7 light otherstructure Objects n04148703 sconce.n.03 28 lighting +242 door handle door handle 12 758 40 7 handle otherprop Objects 40 misc +243 scale scale 12 639 40 7 scale otherprop Objects n04141975 scale.n.07 39 objects +244 trash bin trashcan 12 12 39 6 garbage bin otherfurniture Furniture trash_bin 02747177 n02747177 ashcan.n.01 39 objects +245 baseboard baseboard 13 38 7 otherstructure Objects n02800354 baseboard.n.01 1 wall +246 window /otherroom window /otherroom 12 59 9 13 window window Window n04587648 window.n.01 9 window +247 bag bag 11 55 37 7 bag bag Objects suitcase 02773838 n02773838 bag.n.06 39 objects +248 laptop laptop 11 37 40 7 laptop otherprop Objects laptop laptop 03642806 n03642806 laptop.n.01 39 objects +249 treadmill treadmill 12 458 39 6 treadmill otherfurniture Furniture n04477387 treadmill.n.01 33 gym_equipment +250 staircase staircase 11 215 38 7 stairs otherstructure Objects n04298308 stairway.n.01 16 stairs +251 guitar guitar 11 300 40 7 guitar otherprop Objects guitar guitar 03467517 n03467517 guitar.n.01 39 objects +252 light fixture light fixture 11 62 38 7 light otherstructure Objects n03665366 light.n.02 28 lighting +253 pipes pipe 11 41 40 7 pipe otherprop Objects n03944672 pipe.n.02 40 misc +254 display case display case 11 540 39 6 display case otherfurniture Furniture n02975212 case.n.20 39 objects +255 weight machine exercise equipment 10 457 39 6 excercise equipment otherfurniture Furniture n04285146 sports_equipment.n.01 33 gym_equipment +256 toilet paper holder toilet paper holder 10 647 40 7 toilet paper holder otherprop Objects 40 misc +257 basin basin 10 24 34 7 sink sink Objects sink n04223580 sink.n.01 15 sink +258 towel bar towel bar 10 51 38 7 bar otherstructure Objects n04459909 towel_rail.n.01 39 objects +259 floor behind floor /otherroom 10 11 2 5 floor floor Floor n03365592 floor.n.01 2 floor +260 wooden chair chair 10 5 5 4 chair chair Chair chair chair chair 03001627 n03001627 chair.n.01 3 chair +261 potted plant plant 10 82 40 7 plant otherprop Objects plant n00017222 plant.n.02 14 plant +262 ceiling / other room ceiling /otherroom 10 4 22 3 ceiling ceiling Ceiling n02990373 ceiling.n.01 17 ceiling +263 window sill window frame 10 59 9 13 window window Window n04589593 window_frame.n.01 9 window +264 floor / other room floor /otherroom 10 11 2 5 floor floor Floor n03365592 floor.n.01 2 floor +265 cupboard cabinet 10 3 3 6 cabinet cabinet Furniture cabinet 02933112 n02933112 cabinet.n.01 7 cabinet +266 tray tray 10 179 40 7 tray otherprop Objects n04476259 tray.n.01 39 objects +267 urn urn 10 151 40 7 urn otherprop Objects vase jar 03593526 n04516116 urn.n.01 39 objects +268 church seating pew 10 204 39 6 bench otherfurniture Furniture bench bench 02828884 n03920867 pew.n.01 34 seating +269 decorative plate decorative plate 9 383 40 7 decorative plate otherprop Objects 40 misc +270 doors door 9 28 8 12 door door Wall door n03221720 door.n.01 4 door +271 bar bar 9 51 38 7 bar otherstructure Objects n02788689 bar.n.03 39 objects +272 stair rail banister 9 453 38 7 banister otherstructure Objects n02788148 bannister.n.02 30 railing +273 window shade window shade 9 40 7 otherprop Objects n04590129 window_shade.n.01 32 blinds +274 grass grass 11 82 40 7 plant otherprop Objects plant n12102133 grass.n.01 14 plant +275 pool table pool table 9 515 39 6 pool table otherfurniture Furniture table table table 04379243 n03982430 pool_table.n.01 5 table +276 coat coat 9 324 40 7 jacket otherprop Objects n03057021 coat.n.01 38 clothes +277 trees tree 9 82 40 7 plant otherprop Objects plant n13104059 tree.n.01 14 plant +278 cloth cloth 11 40 7 otherprop Objects n03309808 fabric.n.01 39 objects +279 bottle of soap bottle of soap 9 502 40 7 bottle of soap otherprop Objects 40 misc +280 floor lamp floor lamp 9 144 35 7 lamp lamp Objects lamp lamp 03636649 n03367059 floor_lamp.n.01 28 lighting +281 water cooler water cooler 9 509 39 6 water cooler otherfurniture Furniture n04559166 water_cooler.n.01 39 objects +282 pews pew 9 204 39 6 bench otherfurniture Furniture bench bench 02828884 n03920867 pew.n.01 34 seating +283 ledge ledge 10 38 7 otherstructure Objects n09337253 ledge.n.01 39 objects +284 kitchen shelf kitchen shelf 9 42 15 6 shelves shelves Furniture n04190052 shelf.n.01 31 shelving +285 bathroom utencils bathroom utensil 8 267 40 7 utensil otherprop Objects n04516672 utensil.n.01 39 objects +286 hanger hanger 8 211 40 7 hanger otherprop Objects n03490884 hanger.n.02 39 objects +287 shrubbery shrubbery 8 40 7 otherprop Objects n08649067 shrubbery.n.01 39 objects +288 teapot teapot 8 678 40 7 tea pot otherprop Objects n04398044 teapot.n.01 39 objects +289 bottles bottle 8 2 40 7 bottle otherprop Objects bottle bottle 02876657 n02876657 bottle.n.01 39 objects +290 exercise equipment exercise equipment 8 457 39 6 excercise equipment otherfurniture Furniture n04285146 sports_equipment.n.01 33 gym_equipment +291 boxes box 8 26 29 7 box box Objects n02883344 box.n.01 39 objects +292 locker locker 8 3 3 6 cabinet cabinet Furniture n02933462 cabinet.n.03 40 misc +293 wall cabinet wall cabinet 8 3 3 6 cabinet cabinet Furniture cabinet 02933112 n02933112 cabinet.n.01 7 cabinet +294 wainscotting \other room paneling /otherroom 8 21 1 12 wall wall Wall n03882611 paneling.n.01 1 wall +295 ceiling light ceiling lamp 9 144 35 7 lamp lamp Objects lamp lamp 03636649 n03636649 lamp.n.02 28 lighting +296 ornament ornament 8 40 7 otherprop Objects n03169390 decoration.n.01 39 objects +297 bidet bidet 8 124 33 7 toilet toilet Objects toilet toilet n02836174 bidet.n.01 18 toilet +298 shower soap shelf shower soap shelf 8 40 7 otherprop Objects 40 misc +299 window / door window/door 8 40 7 otherprop Objects 40 misc +300 stuffed animal stuffed animal 8 177 40 7 stuffed animal otherprop Objects n04399382 teddy.n.01 39 objects +301 paper towel dispenser paper towel dispenser 8 14 40 7 paper towel dispenser otherprop Objects 40 misc +302 chair bottom chair 8 5 5 4 chair chair Chair chair chair chair 03001627 n03001627 chair.n.01 3 chair +303 fencing fencing 8 40 7 otherprop Objects n03327234 fence.n.01 40 misc +304 lampshade lampshade 8 859 40 7 lamp shade otherprop Objects n03637318 lampshade.n.01 28 lighting +305 door side door frame 12 28 8 12 door door Wall door n03221720 door.n.01 4 door +306 bust bust 7 294 40 7 sculpture otherprop Objects n02926188 bust.n.03 39 objects +307 car car 7 530 40 7 car otherprop Objects car car 02958343 n02958343 car.n.01 39 objects +308 figure figure 7 40 7 otherprop Objects 40 misc +309 sofa set sofa set 7 83 6 9 sofa sofa Sofa sofa sofa sofa 04256520 n04256520 sofa.n.01 10 sofa +310 commode toilet 7 124 33 7 toilet toilet Objects toilet toilet n04446276 toilet.n.01 18 toilet +311 toilet brush toilet brush 7 630 40 7 toilet brush otherprop Objects 40 misc +312 doll doll 7 99 40 7 doll otherprop Objects n03219135 doll.n.01 39 objects +313 drums drum 7 145 40 7 drum otherprop Objects n03249569 drum.n.01 39 objects +314 bathroom counter bathroom counter 7 7 12 6 counter counter Furniture table table table 04379243 n03116530 counter.n.01 26 counter +315 dress dress 7 40 7 otherprop Objects n03236735 dress.n.01 38 clothes +316 shower handle shower handle 7 758 40 7 handle otherprop Objects 40 misc +317 closet door closet door 7 28 8 12 door door Wall door n03221720 door.n.01 4 door +318 whiteboard whiteboard 7 45 30 7 whiteboard whiteboard Objects n03211616 display_panel.n.01 22 tv_monitor +319 garage door opener garage door opener 7 40 7 otherprop Objects 40 misc +320 range hood range hood 7 380 38 7 range hood otherstructure Objects range_hood n04053677 range_hood.n.01 39 objects +321 window curtain window curtain 7 89 16 13 curtain curtain Window curtain n03151077 curtain.n.01 12 curtain +322 easel easel 7 50 39 6 stand otherfurniture Furniture n03262809 easel.n.01 31 shelving +323 bowl of fruit bowl of fruit 7 22 40 7 bowl otherprop Objects bowl bowl 02880940 n02880940 bowl.n.03 39 objects +324 molding molding 10 38 7 otherstructure Objects n02800354 baseboard.n.01 1 wall +325 pool pool 7 38 7 otherstructure Objects n03982060 pool.n.01 40 misc +326 kitchen appliance kitchen appliance 6 40 7 otherprop Objects n03620052 kitchen_appliance.n.01 37 appliances +327 candelabra candelabra 6 605 40 7 candelabra otherprop Objects n02947818 candelabrum.n.01 39 objects +328 ceiling lamp ceiling lamp 6 144 35 7 lamp lamp Objects lamp lamp 03636649 n03636649 lamp.n.02 28 lighting +329 toy toy 6 389 40 7 toy otherprop Objects n03964744 plaything.n.01 39 objects +330 top top 6 40 7 otherprop Objects 40 misc +331 wall art picture 6 64 11 8 picture picture Picture n03931044 picture.n.01 6 picture +332 highchair highchair 6 5 5 4 chair chair Chair chair chair chair 03001627 n03518445 highchair.n.01 3 chair +333 footrest footrest 6 163 39 6 foot rest otherfurniture Furniture stool n03380724 footstool.n.01 19 stool +334 bathroom sink sink 6 24 34 7 sink sink Objects sink n04223580 sink.n.01 15 sink +335 soap dish soap dish 6 638 40 7 soap dish otherprop Objects n04254009 soap_dish.n.01 39 objects +336 miscellaneous object object 6 40 7 otherprop Objects n00002684 object.n.01 39 objects +337 trim molding 7 38 7 otherstructure Objects n02800354 baseboard.n.01 1 wall +338 tabletop object object 6 40 7 otherprop Objects n00002684 object.n.01 39 objects +339 clothes hanger rod clothes hanger rod 6 40 7 otherprop Objects n04100174 rod.n.01 39 objects +340 altar altar 6 19 7 10 table table Table table table table 04379243 n02699629 altar.n.01 5 table +341 candles candle 6 137 40 7 candle otherprop Objects lamp n02948072 candle.n.01 39 objects +342 placemat place mat 6 154 40 7 placemat otherprop Objects n03952886 place_mat.n.01 39 objects +343 kitchen center island kitchen island 6 456 38 7 kitchen island otherstructure Objects n03620600 kitchen_island.n.01 26 counter +344 plate of food plate of food 6 40 7 otherprop Objects 40 misc +345 sheet sheet 7 559 40 7 sheet otherprop Objects 40 misc +346 wood wood 6 40 7 otherprop Objects 40 misc +347 robe robe 6 40 7 otherprop Objects n04097866 robe.n.01 38 clothes +348 bathroom stall bathroom stall 6 38 7 otherstructure Objects n02873839 booth.n.02 40 misc +349 tabletop decoration decoration 6 40 7 otherprop Objects n03169390 decoration.n.01 39 objects +350 plush toy plush toy 6 389 40 7 toy otherprop Objects n04399382 teddy.n.01 39 objects +351 wash basin washbasin 6 24 34 7 sink sink Objects sink n04553920 washbasin.n.01 15 sink +352 celing ceiling 6 4 22 3 ceiling ceiling Ceiling n02990373 ceiling.n.01 17 ceiling +353 hangers hanger 6 211 40 7 hanger otherprop Objects n03490884 hanger.n.02 39 objects +354 bushes bush 6 82 40 7 plant otherprop Objects plant n13112664 shrub.n.01 14 plant +355 floor/other room floor /otherroom 6 11 2 5 floor floor Floor n03365592 floor.n.01 2 floor +356 dinner placesetting place mat 6 154 40 7 placemat otherprop Objects n03952886 place_mat.n.01 39 objects +357 curtain valence curtain valence 6 89 16 13 curtain curtain Window curtain n03151077 curtain.n.01 12 curtain +358 control control 6 40 7 otherprop Objects n03096960 control.n.09 39 objects +359 tap tap 6 9 40 7 faucet otherprop Objects faucet 03325088 n04559451 water_faucet.n.01 39 objects +360 shampoo shampoo 6 548 40 7 cleaner otherprop Objects n04183516 shampoo.n.01 39 objects +361 computer computer 7 46 40 7 computer otherprop Objects n03082979 computer.n.01 39 objects +362 massage bed massage bed 6 157 4 1 bed bed Bed bed bed bed 02818832 n02818832 bed.n.01 11 bed +363 knob knob 6 652 40 7 knob otherprop Objects 40 misc +364 door stopper door stopper 6 40 7 otherprop Objects 40 misc +365 bulletin board bulletin board 6 408 38 7 board otherstructure Objects n03211616 display_panel.n.01 22 tv_monitor +366 brick archway archway 6 21 1 12 wall wall Wall n02734217 arch.n.03 4 door +367 fruit bowl fruit bowl 6 22 40 7 bowl otherprop Objects bowl bowl 02880940 n02880940 bowl.n.03 39 objects +368 electric wire casing electric wire casing 5 40 7 otherprop Objects 40 misc +369 bookcase bookshelf 5 88 10 6 bookshelf bookshelf Furniture bookshelf bookshelf 02871439 n02871439 bookshelf.n.01 31 shelving +370 other room unknown /otherroom 6 20 40 7 unknown otherprop Objects n08632096 unknown.n.01 41 unlabeled +371 exercise machine exercise equipment 5 457 39 6 excercise equipment otherfurniture Furniture n04285146 sports_equipment.n.01 33 gym_equipment +372 storage shelving storage shelving 5 42 15 6 shelves shelves Furniture n04190052 shelf.n.01 31 shelving +373 coffee maker coffee maker 5 40 7 otherprop Objects n03063338 coffee_maker.n.01 37 appliances +374 shower ceiling shower ceiling 5 4 22 3 ceiling ceiling Ceiling n04208936 shower.n.01 23 shower +375 fire alarm fire alarm 5 338 40 7 fire alarm otherprop Objects n03343737 fire_alarm.n.02 39 objects +376 tissue paper tissue paper 5 15 26 7 paper paper Objects 40 misc +377 projector projector 5 90 40 7 projector otherprop Objects n04009552 projector.n.02 39 objects +378 coat hanger coat hanger 5 400 40 7 coat hanger otherprop Objects n03057920 coat_hanger.n.01 39 objects +379 wall cubby wall cubby 5 40 7 otherprop Objects 40 misc +380 balcony railing balcony railing 5 497 38 7 railing otherstructure Objects 40 misc +381 shelf /w clutter shelf /w clutter 5 42 15 6 shelves shelves Furniture n04190052 shelf.n.01 31 shelving +382 case case 5 851 40 7 case otherprop Objects 40 misc +383 coat rack coat rack 5 40 7 otherprop Objects n03059103 coatrack.n.01 40 misc +384 pan pan 5 589 40 7 pan otherprop Objects n03880531 pan.n.01 39 objects +385 fridge refrigerator 5 17 24 6 refridgerator refridgerator Furniture n04070727 refrigerator.n.01 37 appliances +386 suitcase luggage 5 783 40 7 luggage otherprop Objects n02774630 baggage.n.01 39 objects +387 wardrobe rod closet rod 5 40 7 otherprop Objects n04100174 rod.n.01 39 objects +388 hamper clothes hamper 5 39 40 7 basket otherprop Objects basket 02801938 n03050864 clothes_hamper.n.01 39 objects +389 trinket trinket 5 844 40 7 trinket otherprop Objects n02787435 bangle.n.02 39 objects +390 c;lothes hangers clothes hanger 5 211 40 7 hanger otherprop Objects n03057920 coat_hanger.n.01 39 objects +391 paper paper 5 15 26 7 paper paper Objects n14974264 paper.n.01 39 objects +392 back splash backsplash 5 40 7 otherprop Objects 40 misc +393 chimney chimney 5 702 38 7 chimney otherstructure Objects n03017428 chimney.n.01 40 misc +394 arc arch 5 40 7 otherprop Objects n02733524 arch.n.04 40 misc +395 shower bench shower bench 5 204 39 6 bench otherfurniture Furniture bench bench 02828884 n02828884 bench.n.01 34 seating +396 person person 5 331 31 7 person person Objects person n05217688 person.n.02 39 objects +397 tablet tablet 5 19 7 10 table table Table table table table 04379243 n04379243 table.n.02 5 table +398 exercise mat exercise mat 5 143 20 5 floor mat floor mat Floor n03727946 mat.n.03 33 gym_equipment +399 smoke alarm smoke alarm 5 525 40 7 alarm otherprop Objects n03343737 fire_alarm.n.02 39 objects +400 kitchen utensils kitchen utensil 5 267 40 7 utensil otherprop Objects n03621049 kitchen_utensil.n.01 39 objects +401 weights weight 5 40 7 otherprop Objects n04571292 weight.n.02 33 gym_equipment +402 display cabinet display cabinet 5 3 3 6 cabinet cabinet Furniture cabinet 02933112 n02933112 cabinet.n.01 7 cabinet +403 showcase display cabinet 5 3 3 6 cabinet cabinet Furniture cabinet 02933112 n02933112 cabinet.n.01 7 cabinet +404 bedpost bedpost 5 40 7 otherprop Objects n02821415 bedpost.n.01 11 bed +405 file cabinet file cabinet 5 3 3 6 cabinet cabinet Furniture cabinet 02933112 n02933112 cabinet.n.01 7 cabinet +406 umbrella umbrella 5 203 40 7 umbrella otherprop Objects n04507155 umbrella.n.01 39 objects +407 massage table massage table 5 19 7 10 table table Table table table table 04379243 n04379243 table.n.02 5 table +408 laundry basket laundry basket 5 164 40 7 laundry basket otherprop Objects basket 02801938 n03050864 clothes_hamper.n.01 39 objects +409 jar jar 5 70 40 7 jar otherprop Objects jar 03593526 n03593526 jar.n.01 39 objects +410 exercise bike exercise bike 5 457 39 6 excercise equipment otherfurniture Furniture n03302671 exercise_bike.n.01 33 gym_equipment +411 hose hose 5 40 7 otherprop Objects n03539875 hose.n.03 40 misc +412 window dormer window dormer 5 40 7 otherprop Objects 40 misc +413 closet shelf closet shelf 5 40 7 otherprop Objects n04190052 shelf.n.01 31 shelving +414 power breaker box power breaker box 5 26 29 7 box box Objects 40 misc +415 smoke detector smoke detector 5 40 7 otherprop Objects 40 misc +416 door knob door knob 4 27 40 7 door knob otherprop Objects 40 misc +417 shelf side shelf 4 42 15 6 shelves shelves Furniture n04190052 shelf.n.01 31 shelving +418 jacuzzi jacuzzi 4 40 7 otherprop Objects 40 misc +419 backpack backpack 4 206 40 7 backpack otherprop Objects n02769748 backpack.n.01 39 objects +420 wooden desk desk 4 36 14 10 desk desk Table desk desk table 04379243 n03179701 desk.n.01 5 table +421 bath mat bath mat 4 40 7 otherprop Objects n02807401 bath_mat.n.01 2 floor +422 unknown clutter unknown clutter 4 40 7 otherprop Objects 40 misc +423 hook hook 4 40 7 otherprop Objects 40 misc +424 sauna wall wall 4 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +425 elevator elevator 4 40 7 otherprop Objects 40 misc +426 tool tool 4 40 7 otherprop Objects n04451818 tool.n.01 39 objects +427 recliner recliner 4 5 5 4 chair chair Chair chair chair chair 03001627 n04062428 recliner.n.01 3 chair +428 recessed wall recessed wall 4 21 1 12 wall wall Wall 40 misc +429 closet bar closet rod 4 40 7 otherprop Objects n04100174 rod.n.01 39 objects +430 tank tank 4 40 7 otherprop Objects 40 misc +431 toaster toaster 4 251 40 7 toaster otherprop Objects n04442312 toaster.n.02 37 appliances +432 toilet brush holder toilet brush holder 4 40 7 otherprop Objects 40 misc +433 handrail /otherroom handrail /otherroom 4 453 38 7 banister otherstructure Objects n02788148 bannister.n.02 30 railing +434 landing landing 6 40 7 otherprop Objects n03638511 landing.n.01 2 floor +435 book rack book rack 4 224 39 6 bookrack otherfurniture Furniture 40 misc +436 wall mirror mirror 4 122 19 7 mirror mirror Objects n03773035 mirror.n.01 21 mirror +437 hunting trophy hunting trophy 4 547 40 7 trophy otherprop Objects 40 misc +438 rod rod 4 40 7 otherprop Objects n04100174 rod.n.01 39 objects +439 floor mat floor mat 4 143 20 5 floor mat floor mat Floor n03727837 mat.n.01 2 floor +440 motion detector motion detector 4 40 7 otherprop Objects 40 misc +441 clothing clothes 4 141 21 7 clothes clothes Objects n02728440 apparel.n.01 38 clothes +442 can of paint can of paint 4 40 7 otherprop Objects 40 misc +443 medicine cabinet medicine cabinet 4 3 3 6 cabinet cabinet Furniture cabinet 02933112 n03742115 medicine_chest.n.01 7 cabinet +444 sensor sensor 4 40 7 otherprop Objects n03180969 detector.n.01 39 objects +445 cart cart 4 305 40 7 cart otherprop Objects n03484083 handcart.n.01 39 objects +446 slab slab 4 38 7 otherstructure Objects n04233405 slab.n.01 39 objects +447 bean bag chair bean bag chair 4 5 5 4 chair chair Chair chair chair chair 03001627 n03001627 chair.n.01 3 chair +448 window valence window valence 4 89 16 13 curtain curtain Window curtain n03151077 curtain.n.01 12 curtain +449 window /outside window /outside 4 59 9 13 window window Window n04587648 window.n.01 9 window +450 pole pole 4 40 7 otherprop Objects n03976657 pole.n.01 39 objects +451 picture / other room picture /otherroom 4 64 11 8 picture picture Picture n03931044 picture.n.01 6 picture +452 canister canister 4 794 40 7 canister otherprop Objects 40 misc +453 washbasin counter washbasin counter 4 7 12 6 counter counter Furniture table table table 04379243 n03116530 counter.n.01 26 counter +454 kitchen top kitchen top 4 7 12 6 counter counter Furniture n03118245 countertop.n.01 26 counter +455 unknown object object 4 40 7 otherprop Objects n00002684 object.n.01 39 objects +456 pitcher pitcher 4 273 40 7 pitcher otherprop Objects n03950228 pitcher.n.02 39 objects +457 showerhead showerhead 4 650 40 7 shower head otherprop Objects n04209383 showerhead.n.01 23 shower +458 podium podium 4 40 7 otherprop Objects n03159640 dais.n.01 39 objects +459 ceiling vent ceiling vent 4 25 38 7 air vent otherstructure Objects n04526241 vent.n.01 40 misc +460 throw pillow pillow 4 119 18 7 pillow pillow Objects pillow 03938244 n03938244 pillow.n.01 8 cushion +461 grill grill 4 700 38 7 grill otherstructure Objects n03459591 grill.n.02 40 misc +462 sink cabinet sink cabinet 4 3 3 6 cabinet cabinet Furniture cabinet 02933112 n02933112 cabinet.n.01 7 cabinet +463 tapestry tapestry 4 40 7 otherprop Objects n04393549 tapestry.n.02 39 objects +464 bed sheet bed sheet 5 559 40 7 sheet otherprop Objects n04188179 sheet.n.03 11 bed +465 shade shade 4 40 7 otherprop Objects n04181718 shade.n.03 39 objects +466 doorknob doorknob 4 27 40 7 door knob otherprop Objects n03222959 doorknob.n.01 4 door +467 stair railing banister 4 453 38 7 banister otherstructure Objects n02788148 bannister.n.02 30 railing +468 vacuum cleaner vacuum cleaner 4 306 40 7 vacuum cleaner otherprop Objects n04517823 vacuum.n.04 37 appliances +469 bed comforter bed comforter 4 484 40 7 comforter otherprop Objects 40 misc +470 door / other room door /otherroom 4 28 8 12 door door Wall door n03221720 door.n.01 4 door +471 pictures picture 4 64 11 8 picture picture Picture n03931044 picture.n.01 6 picture +472 bed table bed table 4 19 7 10 table table Table table table table 04379243 n04379243 table.n.02 5 table +473 shirt shirt 4 40 7 otherprop Objects n04197391 shirt.n.01 38 clothes +474 dressing table dressing table 4 19 7 10 table table Table table table table 04379243 n03238586 dressing_table.n.01 5 table +475 shower wall cubby shower wall cubby 4 40 7 otherprop Objects n04208936 shower.n.01 23 shower +476 side wall wall 4 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +477 beside table beside table 4 19 7 10 table table Table table table table 04379243 n04379243 table.n.02 5 table +478 curb curb 4 40 7 otherprop Objects n03149135 curb.n.02 39 objects +479 arches arch 4 40 7 otherprop Objects n02733524 arch.n.04 40 misc +480 storage bin storage bin 4 812 40 7 storage bin otherprop Objects 40 misc +481 support beam support beam 4 40 7 otherprop Objects n02815950 beam.n.02 29 beam +482 globe globe 5 347 40 7 globe otherprop Objects 40 misc +483 pantry pantry 4 38 7 otherstructure Objects n03885535 pantry.n.01 40 misc +484 skateboard skateboard 4 408 38 7 board otherstructure Objects skateboard 04225987 n04225987 skateboard.n.01 39 objects +485 stove hood range hood 4 380 38 7 range hood otherstructure Objects range_hood n04053677 range_hood.n.01 39 objects +486 cabin cabin 4 40 7 otherprop Objects 40 misc +487 shower bar shower bar 4 51 38 7 bar otherstructure Objects 40 misc +488 chaise chaise 4 5 5 4 chair chair Chair chair chair chair 03001627 n03002711 chaise_longue.n.01 3 chair +489 flower wash flower vase 4 78 40 7 vase otherprop Objects vase jar 03593526 n04522168 vase.n.01 39 objects +490 desk and chairs desk and chairs 4 40 7 otherprop Objects 40 misc +491 plant vase flower vase 4 78 40 7 vase otherprop Objects vase jar 03593526 n04522168 vase.n.01 39 objects +492 towel rack towel rack 4 40 7 otherprop Objects n04459773 towel_rack.n.01 40 misc +493 cross cross 4 49 40 7 monitor otherprop Objects n03857828 oscilloscope.n.01 39 objects +494 sliding door sliding door 4 28 8 12 door door Wall door n04239074 sliding_door.n.01 4 door +495 cosmetics cosmetics 4 40 7 otherprop Objects n03113152 cosmetic.n.01 39 objects +496 wall other room wall /otherroom 4 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +497 kettle kettle 4 16 40 7 pot otherprop Objects n03612814 kettle.n.01 39 objects +498 junk junk 4 40 7 otherprop Objects n14857897 debris.n.01 39 objects +499 stationery stationery 3 15 26 7 paper paper Objects n06258852 stationery.n.01 39 objects +500 lights light 3 144 35 7 lamp lamp Objects lamp lamp 03636649 n03636649 lamp.n.02 28 lighting +501 window / other room window /otherroom 3 59 9 13 window window Window n04587648 window.n.01 9 window +502 dish cabinet dish cabinet 3 3 3 6 cabinet cabinet Furniture cabinet 02933112 n02933112 cabinet.n.01 7 cabinet +503 wall panel wall panel 3 21 1 12 wall wall Wall n04548503 wall_panel.n.01 1 wall +504 gate gate 3 223 38 7 gate otherstructure Objects n03427296 gate.n.01 40 misc +505 safe safe 3 26 29 7 box box Objects n04125021 safe.n.01 39 objects +506 ventilation ventilation 3 40 7 otherprop Objects n04526520 ventilation.n.02 39 objects +507 logs firewood 3 40 7 otherprop Objects n15100644 firewood.n.01 40 misc +508 sliding doors sliding door 3 28 8 12 door door Wall door n04239074 sliding_door.n.01 4 door +509 shower rod shower rod 3 40 7 otherprop Objects n04100174 rod.n.01 12 curtain +510 towel shelf towel shelf 3 42 15 6 shelves shelves Furniture n04190052 shelf.n.01 31 shelving +511 row of theater chairs row of theater chairs 3 40 7 otherprop Objects 40 misc +512 closet wall wall 3 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +513 bath utencils bath utensil 3 267 40 7 utensil otherprop Objects 40 misc +514 sconce sconce 3 62 38 7 light otherstructure Objects n04148703 sconce.n.03 28 lighting +515 garage light garage light 3 62 38 7 light otherstructure Objects 40 misc +516 ceiling beam beam 3 38 7 otherstructure Objects n02815950 beam.n.02 29 beam +517 toolbox toolbox 3 344 39 6 chest otherfurniture Furniture n04452615 toolbox.n.01 39 objects +518 sponge stool stool 3 150 40 7 stool otherprop Objects stool n04326896 stool.n.01 19 stool +519 security camera security camera 3 212 40 7 security camera otherprop Objects camera 02942699 n02942699 camera.n.01 39 objects +520 gym machine exercise equipment 3 457 39 6 excercise equipment otherfurniture Furniture n04285146 sports_equipment.n.01 33 gym_equipment +521 wall /outside wall /outside 3 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +522 mantle mantle 3 874 38 7 mantle otherstructure Objects mantel n03719343 mantel.n.01 27 fireplace +523 floor 2 floor 3 11 2 5 floor floor Floor n03365592 floor.n.01 2 floor +524 skirting skirting 3 40 7 otherprop Objects 40 misc +525 banister banister 3 453 38 7 banister otherstructure Objects n02788148 bannister.n.02 30 railing +526 lockers locker 3 3 3 6 cabinet cabinet Furniture n02933462 cabinet.n.03 40 misc +527 trophy trophy 3 547 40 7 trophy otherprop Objects 40 misc +528 tile tile 3 40 7 otherprop Objects n04435180 tile.n.01 39 objects +529 unknown / room below unknown /otherroom 3 20 40 7 unknown otherprop Objects n08632096 unknown.n.01 41 unlabeled +530 picture car picture car 3 530 40 7 car otherprop Objects car car 02958343 n02958343 car.n.01 39 objects +531 outlet outlet 3 40 7 otherprop Objects n04548771 wall_socket.n.01 39 objects +532 plant ridge plant ridge 3 40 7 otherprop Objects 40 misc +533 backsplash backsplash 3 40 7 otherprop Objects 40 misc +534 computer monitor monitor 3 49 40 7 monitor otherprop Objects monitor monitor tv or monitor 03211117 n03782190 monitor.n.04 22 tv_monitor +535 doorframe / other room doorframe /otherroom 3 615 38 7 door frame otherstructure Objects n03222722 doorframe.n.01 4 door +536 shower wall /otherroom shower wall /otherroom 3 21 1 12 wall wall Wall n04208936 shower.n.01 23 shower +537 shelves with wine shelving 3 42 15 6 shelves shelves Furniture n04190052 shelf.n.01 31 shelving +538 shoe shoe 3 149 40 7 shoe otherprop Objects n04199027 shoe.n.01 39 objects +539 hedge hedge 3 40 7 otherprop Objects n03511175 hedge.n.01 40 misc +540 glass window window 3 59 9 13 window window Window n04587648 window.n.01 9 window +541 sofachair sofa chair 3 5 5 4 chair chair Chair chair chair chair 03001627 n03001627 chair.n.01 3 chair +542 hand soap hand soap 3 133 40 7 soap otherprop Objects n04253437 soap.n.01 39 objects +543 window blinds blinds 3 80 13 13 blinds blinds Window n04589190 window_blind.n.01 32 blinds +544 flower vase flower vase 3 78 40 7 vase otherprop Objects vase jar 03593526 n04522168 vase.n.01 39 objects +545 appliance appliance 3 40 7 otherprop Objects 40 misc +546 christmas tree christmas tree 3 40 7 otherprop Objects 40 misc +547 support column column 3 94 38 7 column otherstructure Objects n03074380 column.n.06 40 misc +548 dish plate 3 233 40 7 plate otherprop Objects n03959485 plate.n.04 39 objects +549 closet floor closet floor 3 11 2 5 floor floor Floor 40 misc +550 celling ceiling 3 4 22 3 ceiling ceiling Ceiling n02990373 ceiling.n.01 17 ceiling +551 stuffed animals stuffed animal 3 177 40 7 stuffed animal otherprop Objects n04399382 teddy.n.01 39 objects +552 casket casket 3 40 7 otherprop Objects 40 misc +553 rowing machine exercise machine 3 220 40 7 machine otherprop Objects 40 misc +554 dining table centerpiece dining table centerpiece 3 878 40 7 centerpiece otherprop Objects 40 misc +555 bedside lamp bedside lamp 5 144 35 7 lamp lamp Objects lamp lamp 03636649 n03636649 lamp.n.02 28 lighting +556 kitchen countertop item kitchen countertop item 3 40 7 otherprop Objects 40 misc +557 fountain fountain 3 40 7 otherprop Objects n03388043 fountain.n.01 40 misc +558 window glass window glass 3 612 38 7 glass otherstructure Objects n03881893 pane.n.01 39 objects +559 wall soffet soffit 3 40 7 otherprop Objects n04256758 soffit.n.01 39 objects +560 urinal urinal 3 40 7 otherprop Objects n04515991 urinal.n.01 39 objects +561 kitchen hood range hood 3 380 38 7 range hood otherstructure Objects range_hood n04053677 range_hood.n.01 39 objects +562 showpiece decoration 3 40 7 otherprop Objects n03169390 decoration.n.01 39 objects +563 wall trim molding 3 38 7 otherstructure Objects n02800354 baseboard.n.01 1 wall +564 barrel barrel 3 343 39 6 barrel otherfurniture Furniture 40 misc +565 firewood firewood 3 40 7 otherprop Objects n15100644 firewood.n.01 40 misc +566 columns pillar 3 94 38 7 column otherstructure Objects n03073977 column.n.07 24 column +567 beams beam 3 38 7 otherstructure Objects n02815950 beam.n.02 29 beam +568 carpet roll carpet roll 3 40 7 otherprop Objects 40 misc +569 portrait portrait 3 64 11 8 picture picture Picture n03987079 portrait.n.02 6 picture +570 table light table light 3 62 38 7 light otherstructure Objects 40 misc +571 water heater water heater 3 588 40 7 water heater otherprop Objects n04560113 water_heater.n.01 39 objects +572 stairs 2 stair 3 215 38 7 stairs otherstructure Objects stairs n04314914 step.n.04 16 stairs +573 pouffe pouffe 3 359 39 6 ottoman otherfurniture Furniture n03858418 ottoman.n.03 34 seating +574 ceiling behind ceiling /otherroom 3 4 22 3 ceiling ceiling Ceiling n02990373 ceiling.n.01 17 ceiling +575 pillars pillar 3 94 38 7 column otherstructure Objects n03073977 column.n.07 24 column +576 concrete block concrete block 3 40 7 otherprop Objects 40 misc +577 range stove 3 242 38 7 stove otherstructure Objects stove 04330267 n04330267 stove.n.02 37 appliances +578 shelf with objects shelf 3 42 15 6 shelves shelves Furniture n04190052 shelf.n.01 31 shelving +579 toilet seat liner dispenser toilet seat liner dispenser 3 40 7 otherprop Objects 40 misc +580 patio chair patio chair 3 5 5 4 chair chair Chair chair chair chair 03001627 n03001627 chair.n.01 3 chair +581 folding chair folding chair 3 5 5 4 chair chair Chair chair chair chair 03001627 n03376595 folding_chair.n.01 3 chair +582 island island 4 456 38 7 kitchen island otherstructure Objects n03620600 kitchen_island.n.01 26 counter +583 toaster oven toaster oven 3 275 40 7 toaster oven otherprop Objects n04442441 toaster_oven.n.01 37 appliances +584 overlook railing railing 3 497 38 7 railing otherstructure Objects n04047401 railing.n.01 30 railing +585 bathroom mirror mirror 3 122 19 7 mirror mirror Objects n03773035 mirror.n.01 21 mirror +586 recycle bin recycle bin 3 307 40 7 bin otherprop Objects 40 misc +587 counter top countertop 3 7 12 6 counter counter Furniture n03118245 countertop.n.01 26 counter +588 rafter rafter 3 40 7 otherprop Objects n04045644 rafter.n.01 29 beam +589 dryer clothes dryer 3 39 6 otherfurniture Furniture n03251766 dryer.n.01 37 appliances +590 bed lamp bedside lamp 3 144 35 7 lamp lamp Objects lamp lamp 03636649 n03636649 lamp.n.02 28 lighting +591 wall paneling paneling 3 21 1 12 wall wall Wall n03882611 paneling.n.01 1 wall +592 stage stage 3 40 7 otherprop Objects 40 misc +593 fire sprinkler fire sprinkler 3 40 7 otherprop Objects 40 misc +594 brush brush 3 40 7 otherprop Objects n02908217 brush.n.02 39 objects +595 wall 2 wall 3 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +596 balcony balcony 3 40 7 otherprop Objects 40 misc +597 water tank water tank 2 263 40 7 vessel otherprop Objects n03035715 cistern.n.02 39 objects +598 pile of objects clutter 2 40 7 otherprop Objects 40 misc +599 garage door frame garage door frame 2 40 7 otherprop Objects 40 misc +600 back wall wall 2 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +601 plant soil plant soil 2 40 7 otherprop Objects 40 misc +602 globe stand globe stand 2 466 40 7 globe stand otherprop Objects 40 misc +603 bicycle bicycle 2 189 40 7 bicycle otherprop Objects bicycle 02834778 n02834778 bicycle.n.01 39 objects +604 stairframe stair frame 2 40 7 otherprop Objects 40 misc +605 separating screen partition 2 21 1 12 wall wall Wall n03894379 partition.n.01 40 misc +606 unknown wall object object 2 40 7 otherprop Objects n00002684 object.n.01 39 objects +607 air duct air duct 2 38 38 7 air duct otherstructure Objects n02690941 air_passage.n.01 40 misc +608 kitchen upper cabinet kitchen cabinet 2 3 3 6 cabinet cabinet Furniture n02933112 cabinet.n.01 7 cabinet +609 chair pillow pillow 2 119 18 7 pillow pillow Objects pillow 03938244 n03938244 pillow.n.01 8 cushion +610 led tv led tv 2 40 7 otherprop Objects 40 misc +611 deco decoration 2 40 7 otherprop Objects n03169390 decoration.n.01 39 objects +612 ceiling under staircase ceiling under staircase 2 40 7 otherprop Objects 40 misc +613 chair /w books chair /w books 2 85 23 2 books books Books 40 misc +614 bed back rest headboard 2 161 39 6 headboard otherfurniture Furniture n03502200 headboard.n.01 11 bed +615 giraffe giraffe 2 40 7 otherprop Objects n02439033 giraffe.n.01 39 objects +616 lightswitch light switch 2 301 38 7 light switch otherstructure Objects n04372370 switch.n.01 39 objects +617 doorframe \other room doorframe /otherroom 2 615 38 7 door frame otherstructure Objects n03222722 doorframe.n.01 4 door +618 object / camera? object 2 40 7 otherprop Objects n00002684 object.n.01 39 objects +619 grandfather clock grandfather clock 2 462 39 6 grandfather clock otherfurniture Furniture clock 03046257 n03452594 grandfather_clock.n.01 39 objects +620 jewelry box jewelry box 2 26 29 7 box box Objects 40 misc +621 bottles of wine bottles of wine 2 766 40 7 wine otherprop Objects 40 misc +622 massage table base massage table 2 19 7 10 table table Table table table table 04379243 n04379243 table.n.02 5 table +623 hood range hood 2 380 38 7 range hood otherstructure Objects range_hood n04053677 range_hood.n.01 39 objects +624 mirrorframe mirror frame 2 122 19 7 mirror mirror Objects n03773035 mirror.n.01 21 mirror +625 wall beam wall beam 2 40 7 otherprop Objects 40 misc +626 wooden trim molding 2 38 7 otherstructure Objects n02800354 baseboard.n.01 1 wall +627 stalls stall 2 38 7 otherstructure Objects n02873839 booth.n.02 40 misc +628 partition partition 2 21 1 12 wall wall Wall n03894379 partition.n.01 40 misc +629 dog dog 2 701 40 7 dog otherprop Objects 40 misc +630 valance valance 2 40 7 otherprop Objects n03111296 cornice.n.01 40 misc +631 radio radio 2 188 40 7 radio otherprop Objects radio 40 misc +632 bush bush 2 82 40 7 plant otherprop Objects plant n13112664 shrub.n.01 14 plant +633 row of theater seats row of theater seats 2 40 7 otherprop Objects 40 misc +634 bath utencil bath utensil 2 267 40 7 utensil otherprop Objects 40 misc +635 basket of towels basket of towels 2 40 7 otherprop Objects 40 misc +636 laundry machine washing machine 2 278 39 6 washing machine otherfurniture Furniture washing_machine 04554684 n04554684 washer.n.03 37 appliances +637 mirror /otherroom mirror /otherroom 2 122 19 7 mirror mirror Objects n03773035 mirror.n.01 21 mirror +638 toilet sink toilet sink 2 24 34 7 sink sink Objects sink 40 misc +639 sauna heater sauna heater 2 111 39 6 heater otherfurniture Furniture 40 misc +640 dining bench dining bench 2 204 39 6 bench otherfurniture Furniture bench bench 02828884 n02828884 bench.n.01 34 seating +641 fume cupboard fume cupboard 2 40 7 otherprop Objects 40 misc +642 mouse mouse 2 103 40 7 mouse otherprop Objects n03793489 mouse.n.04 39 objects +643 boiler boiler 2 40 7 otherprop Objects 40 misc +644 hearth hearth 2 372 38 7 fireplace otherstructure Objects 40 misc +645 curtain darker curtain 2 89 16 13 curtain curtain Window curtain n03151077 curtain.n.01 12 curtain +646 round chair round chair 2 5 5 4 chair chair Chair chair chair chair 03001627 n03001627 chair.n.01 3 chair +647 toilet bowl toilet 2 124 33 7 toilet toilet Objects toilet toilet n04446276 toilet.n.01 18 toilet +648 whine rack wine rack 2 299 40 7 wine rack otherprop Objects 40 misc +649 doorstep doorstep 3 28 8 12 door door Wall n03223686 doorsill.n.01 39 objects +650 binder binder 2 399 40 7 binder otherprop Objects 40 misc +651 shower door frame shower door frame 2 40 7 otherprop Objects n04208936 shower.n.01 23 shower +652 bed runner bed runner 2 157 4 1 bed bed Bed n02822579 bedstead.n.01 11 bed +653 cubicle cubicle 2 40 7 otherprop Objects 40 misc +654 fitness device exercise equipment 2 457 39 6 excercise equipment otherfurniture Furniture n04285146 sports_equipment.n.01 33 gym_equipment +655 support support 2 40 7 otherprop Objects 40 misc +656 overhang overhang 2 40 7 otherprop Objects n03864356 overhang.n.01 40 misc +657 electric box electric box 2 550 38 7 electric box otherstructure Objects 40 misc +658 bathrobe bathrobe 3 40 7 otherprop Objects n02807616 bathrobe.n.01 38 clothes +659 door mat doormat 2 143 20 5 floor mat floor mat Floor n03223299 doormat.n.02 2 floor +660 jacket jacket 2 324 40 7 jacket otherprop Objects n03589791 jacket.n.01 38 clothes +661 cabinet table cabinet table 2 19 7 10 table table Table table table table 04379243 n04379243 table.n.02 5 table +662 side frame frame 2 38 7 otherstructure Objects 40 misc +663 wainscoting paneling 2 21 1 12 wall wall Wall n03882611 paneling.n.01 1 wall +664 staircase trim staircase trim 2 40 7 otherprop Objects 40 misc +665 box /w books box /w books 2 85 23 2 books books Books 40 misc +666 nighstand nightstand 2 158 32 6 night stand night stand Furniture night_stand night_stand n03015254 chest_of_drawers.n.01 13 chest_of_drawers +667 window reflection window reflection 2 40 7 otherprop Objects 40 misc +668 pulpit pulpit 2 40 7 otherprop Objects n03159640 dais.n.01 39 objects +669 set of armchairs set of armchairs 2 40 7 otherprop Objects 40 misc +670 fish tank fish tank 2 782 38 7 fish tank otherstructure Objects n02732072 aquarium.n.01 40 misc +671 bathroom countertop objects objects 2 40 7 otherprop Objects n00002684 object.n.01 39 objects +672 concrete shelf shelf 2 42 15 6 shelves shelves Furniture n04190052 shelf.n.01 31 shelving +673 ceiling wall ceiling wall 2 21 1 12 wall wall Wall 40 misc +674 picture \other room picture /otherroom 2 64 11 8 picture picture Picture n03931044 picture.n.01 6 picture +675 wall entry wall 2 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +676 lintel lintel 2 40 7 otherprop Objects n03503233 header.n.02 29 beam +677 wall table wall table 2 19 7 10 table table Table table table table 04379243 n04379243 table.n.02 5 table +678 small table table 3 19 7 10 table table Table table table table 04379243 n04379243 table.n.02 5 table +679 lighting fixture lighting fixture 2 40 7 otherprop Objects n03667380 lighting_fixture.n.01 39 objects +680 bed frame bedframe 2 157 4 1 bed bed Bed n02822579 bedstead.n.01 11 bed +681 freezer freezer 2 17 24 6 refridgerator refridgerator Furniture n03170635 deep-freeze.n.01 37 appliances +682 glass doors door 2 28 8 12 door door Wall door n03221720 door.n.01 4 door +683 extractor extractor 2 40 7 otherprop Objects 40 misc +684 flower pot flowerpot 2 146 40 7 flower pot otherprop Objects flower_pot flower pot 03991062 n03991062 pot.n.04 39 objects +685 soffet soffit 2 40 7 otherprop Objects n04256758 soffit.n.01 39 objects +686 platform platform 2 38 7 otherstructure Objects 40 misc +687 hot tub hot tub 2 136 36 7 bathtub bathtub Objects bathtub bathtub tub 02808440 n03543603 hot_tub.n.01 25 bathtub +688 paper towels paper towel 2 113 40 7 paper towel otherprop Objects n03887697 paper_towel.n.01 20 towel +689 kitchen utencils kitchen utensil 2 267 40 7 utensil otherprop Objects n03621049 kitchen_utensil.n.01 39 objects +690 shower grab bar shower grab bar 2 51 38 7 bar otherstructure Objects 40 misc +691 wall detail wall detail 2 40 7 otherprop Objects 40 misc +692 whineshelf whine shelf 2 40 7 otherprop Objects 40 misc +693 painting/other room painting /otherroom 2 64 11 8 picture picture Picture n03876519 painting.n.01 39 objects +694 television wall tv 2 40 7 otherprop Objects 40 misc +695 sauna bench bench 2 204 39 6 bench otherfurniture Furniture bench bench 02828884 n02828884 bench.n.01 34 seating +696 fruits fruit 2 286 40 7 fruit otherprop Objects n13134947 fruit.n.01 39 objects +697 picture frames picture frame 2 64 11 8 picture picture Picture n03931765 picture_frame.n.01 6 picture +698 buffet buffet 2 7 12 6 counter counter Furniture table table table 04379243 n04247736 snack_bar.n.01 26 counter +699 billow billow 2 40 7 otherprop Objects 40 misc +700 computer chair computer chair 2 5 5 4 chair chair Chair chair chair chair 03001627 n03001627 chair.n.01 3 chair +701 sofa seat sofa seat 2 40 7 otherprop Objects 40 misc +702 wall tv wall tv 2 40 7 otherprop Objects 40 misc +703 ground floor 2 11 2 5 floor floor Floor n03365592 floor.n.01 2 floor +704 wall2 wall 2 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +705 ceiling under stairs ceiling under stairs 2 215 38 7 stairs otherstructure Objects stairs 40 misc +706 calendar calendar 2 583 40 7 calendar otherprop Objects 40 misc +707 dome dome 2 40 7 otherprop Objects 40 misc +708 object /outside object /outside 2 40 7 otherprop Objects n00002684 object.n.01 39 objects +709 poll poll 2 40 7 otherprop Objects n01817346 poll.n.04 39 objects +710 wet bar wet bar 2 51 38 7 bar otherstructure Objects table table table 04379243 n04573513 wet_bar.n.01 26 counter +711 folding table folding table 2 19 7 10 table table Table table table table 04379243 n04379243 table.n.02 5 table +712 stovetop stovetop 2 40 7 otherprop Objects 40 misc +713 gym equip gym equipment 2 457 39 6 excercise equipment otherfurniture Furniture n04285146 sports_equipment.n.01 33 gym_equipment +714 fruitbowl fruit bowl 2 22 40 7 bowl otherprop Objects bowl bowl 02880940 n02880940 bowl.n.03 39 objects +715 vending machine vending machine 2 220 40 7 machine otherprop Objects n04525305 vending_machine.n.01 39 objects +716 handwash hand wash 2 40 7 otherprop Objects 40 misc +717 wall clock wall clock 2 56 40 7 clock otherprop Objects clock 03046257 n04548280 wall_clock.n.01 39 objects +718 liquid soap liquid soap 2 133 40 7 soap otherprop Objects 40 misc +719 trinkets trinket 2 844 40 7 trinket otherprop Objects n02787435 bangle.n.02 39 objects +720 small table / stand small table/stand 2 40 7 otherprop Objects 40 misc +721 window shuts window shutters 2 40 7 otherprop Objects n04211356 shutter.n.02 39 objects +722 door frame 3 door frame 2 28 8 12 door door Wall door n03221720 door.n.01 4 door +723 stone stone 2 578 40 7 stones otherprop Objects n09416076 rock.n.01 39 objects +724 tripod tripod 2 50 39 6 stand otherfurniture Furniture n04485082 tripod.n.01 31 shelving +725 projector screen projector screen 2 53 38 7 projector screen otherstructure Objects 40 misc +726 window frame / other room window frame /otherroom 2 477 38 7 window frame otherstructure Objects n04589593 window_frame.n.01 9 window +727 wreath wreath 2 881 40 7 wreathe otherprop Objects n04606014 wreath.n.01 39 objects +728 door hinge door hinge 2 40 7 otherprop Objects 40 misc +729 sound speaker 2 54 40 7 speaker otherprop Objects speaker 03691459 n03691459 loudspeaker.n.01 39 objects +730 french door french door 2 28 8 12 door door Wall door n03394649 french_door.n.01 4 door +731 staircase handrail staircase handrail 2 40 7 otherprop Objects 40 misc +732 nighttable night table 2 19 7 10 table table Table table table table 04379243 n04379243 table.n.02 5 table +733 photo in frame picure 2 64 11 8 picture picture Picture 40 misc +734 photoframe picture 2 64 11 8 picture picture Picture n03931044 picture.n.01 6 picture +735 stair wall stair wall 2 21 1 12 wall wall Wall 40 misc +736 closet ceiling closet ceiling 2 4 22 3 ceiling ceiling Ceiling 40 misc +737 stick stick 2 529 40 7 stick otherprop Objects 40 misc +738 fluorescent light fluorescent light 2 62 38 7 light otherstructure Objects 40 misc +739 wash cabinet wash cabinet 2 3 3 6 cabinet cabinet Furniture cabinet 02933112 n02933112 cabinet.n.01 7 cabinet +740 shower seat shower seat 2 40 7 otherprop Objects 40 misc +741 trellis trellis 2 40 7 otherprop Objects n04478512 trellis.n.01 40 misc +742 patio patio 2 40 7 otherprop Objects n03899768 patio.n.01 40 misc +743 dart board dartboard 2 408 38 7 board otherstructure Objects n03162940 dartboard.n.01 39 objects +744 comforter comforter 2 484 40 7 comforter otherprop Objects n04033995 quilt.n.01 39 objects +745 table /w books table /w books 2 85 23 2 books books Books 40 misc +746 picture/other room picture /otherroom 2 64 11 8 picture picture Picture n03931044 picture.n.01 6 picture +747 dirt dirt 2 40 7 otherprop Objects 40 misc +748 base base 2 40 7 otherprop Objects 40 misc +749 chemical tank chemical tank 2 40 7 otherprop Objects 40 misc +750 step stool step stool 2 276 40 7 step stool otherprop Objects stool n04315713 step_stool.n.01 19 stool +751 misc misc 2 40 7 otherprop Objects 40 misc +752 sink table sink table 2 19 7 10 table table Table table table table 04379243 n04379243 table.n.02 5 table +753 curved wall wall 2 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +754 roof beam beam 2 38 7 otherstructure Objects n02815950 beam.n.02 29 beam +755 cover cover 2 312 40 7 blanket otherprop Objects 40 misc +756 reading table reading table 2 19 7 10 table table Table table table table 04379243 n04379243 table.n.02 5 table +757 side steps wall steps wall 2 21 1 12 wall wall Wall 40 misc +758 sideboard sideboard 2 7 12 6 counter counter Furniture 40 misc +759 wall from another room wall /otherroom 2 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +760 electric outlet electric outlet 2 98 40 7 electrical outlet otherprop Objects n04548771 wall_socket.n.01 39 objects +761 stall door stall door 2 28 8 12 door door Wall door n03221720 door.n.01 4 door +762 separator separator 2 40 7 otherprop Objects n02995998 centrifuge.n.01 39 objects +763 toilet bowl brush holder toilet bowl brush holder 2 40 7 otherprop Objects 40 misc +764 vessel vessel 2 263 40 7 vessel otherprop Objects watercraft 04530566 n04530566 vessel.n.02 39 objects +765 table clutter table clutter 2 40 7 otherprop Objects 40 misc +766 partition wall wall 2 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +767 bed curtain bed curtain 2 89 16 13 curtain curtain Window curtain 40 misc +768 stairs skirt stairs skirt 2 40 7 otherprop Objects 40 misc +769 small chair chair 2 5 5 4 chair chair Chair chair chair chair 03001627 n03001627 chair.n.01 3 chair +770 ceiling window ceiling window 2 59 9 13 window window Window 40 misc +771 rocking chair rocking chair 2 5 5 4 chair chair Chair chair chair chair 03001627 n04099969 rocking_chair.n.01 3 chair +772 window \other room window /otherroom 2 59 9 13 window window Window n04587648 window.n.01 9 window +773 under stair under stair 2 40 7 otherprop Objects 40 misc +774 unknown outside building unknown /outside 2 20 40 7 unknown otherprop Objects n08632096 unknown.n.01 41 unlabeled +775 bath wall bath wall 2 21 1 12 wall wall Wall 40 misc +776 panel screen panel screen 2 40 7 otherprop Objects 40 misc +777 doorfra,e doorframe 2 615 38 7 door frame otherstructure Objects n03222722 doorframe.n.01 4 door +778 shower mat shower mat 2 143 20 5 floor mat floor mat Floor n03727837 mat.n.01 2 floor +779 blackboard blackboard 2 225 38 7 blackboard otherstructure Objects n02846511 blackboard.n.01 39 objects +780 drawer desk drawer desk 2 36 14 10 desk desk Table desk desk 40 misc +781 poster picture 2 64 11 8 picture picture Picture n03931044 picture.n.01 6 picture +782 fireplace sconce fireplace sconce 2 40 7 otherprop Objects 40 misc +783 table support table support 2 40 7 otherprop Objects 40 misc +784 wall lamp wall lamp 2 144 35 7 lamp lamp Objects lamp lamp 03636649 n03636649 lamp.n.02 28 lighting +785 closet shelving closet shelving 5 40 7 otherprop Objects 40 misc +786 closest area closest area 2 40 7 otherprop Objects 40 misc +787 scroll scroll 2 40 7 otherprop Objects 40 misc +788 foot stand foot stand 2 50 39 6 stand otherfurniture Furniture 40 misc +789 button button 2 774 40 7 button otherprop Objects 40 misc +790 art / clutter art/clutter 2 40 7 otherprop Objects 40 misc +791 door arc arch 2 40 7 otherprop Objects n02733524 arch.n.04 40 misc +792 stairs railing stairs railing 2 497 38 7 railing otherstructure Objects 40 misc +793 floor /otherroom floor /otherroom 2 11 2 5 floor floor Floor n03365592 floor.n.01 2 floor +794 shovel shovel 2 607 40 7 shovel otherprop Objects 40 misc +795 alarm controls alarm control 2 40 7 otherprop Objects 40 misc +796 lamb lamp 2 144 35 7 lamp lamp Objects lamp lamp 03636649 n03636649 lamp.n.02 28 lighting +797 cluttered objects clutter 2 40 7 otherprop Objects 40 misc +798 door/other room door /otherroom 2 28 8 12 door door Wall door n03221720 door.n.01 4 door +799 closet shelves closet shelf 2 40 7 otherprop Objects n04190052 shelf.n.01 31 shelving +800 scultpure sculpture 2 294 40 7 sculpture otherprop Objects n04157320 sculpture.n.01 39 objects +801 arm chair armchair 2 5 5 4 chair chair Chair chair chair chair 03001627 n02738535 armchair.n.01 3 chair +802 door behind door /otherroom 2 28 8 12 door door Wall door n03221720 door.n.01 4 door +803 exercise ball exercise ball 2 457 39 6 excercise equipment otherfurniture Furniture n04285146 sports_equipment.n.01 33 gym_equipment +804 yard yard 2 40 7 otherprop Objects 40 misc +805 semi chair semi chair 2 5 5 4 chair chair Chair chair chair chair 03001627 n03001627 chair.n.01 3 chair +806 bouquet bouquet 2 40 7 otherprop Objects n02879087 bouquet.n.01 39 objects +807 garage door opener bar garage door opener bar 2 51 38 7 bar otherstructure Objects 40 misc +808 pots pot 2 16 40 7 pot otherprop Objects n03991062 pot.n.04 39 objects +809 decorations decoration 2 40 7 otherprop Objects n03169390 decoration.n.01 39 objects +810 unkown unknown 2 20 40 7 unknown otherprop Objects n08632096 unknown.n.01 41 unlabeled +811 kitchen decoration kitchen decoration 2 40 7 otherprop Objects n03169390 decoration.n.01 39 objects +812 archway corner archway corner 2 40 7 otherprop Objects 40 misc +813 kitchen wall kitchen wall 2 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +814 cabinet door cabinet door 2 28 8 12 door door Wall door 40 misc +815 sauna bowl sauna bowl 2 22 40 7 bowl otherprop Objects bowl bowl 02880940 n02880940 bowl.n.03 39 objects +816 bar chair bar chair 2 5 5 4 chair chair Chair chair chair chair 03001627 n03001627 chair.n.01 3 chair +817 shelf cubby shelf cubby 2 40 7 otherprop Objects 40 misc +818 toilet plunger toilet plunger 2 563 40 7 toilet plunger otherprop Objects n03970156 plunger.n.03 39 objects +819 belts belt 2 610 40 7 belt otherprop Objects 40 misc +820 sewing machine sewing machine 2 890 40 7 sewing machine otherprop Objects n04179913 sewing_machine.n.01 37 appliances +821 hot water/cold water knob hot water/cold water knob 2 652 40 7 knob otherprop Objects 40 misc +822 barbeque barbecue 2 40 7 otherprop Objects n02790669 barbecue.n.03 40 misc +823 cutting board cutting board 2 247 40 7 cutting board otherprop Objects n03025513 chopping_board.n.01 39 objects +824 soapbox soapbox 2 671 40 7 soap box otherprop Objects 40 misc +825 washing stuff washing stuff 2 40 7 otherprop Objects 40 misc +826 dining table decoration decoration 2 40 7 otherprop Objects n03169390 decoration.n.01 39 objects +827 copier copier 2 40 7 otherprop Objects n03257586 duplicator.n.01 39 objects +828 unknown (picture or window) unknown picture/window 2 40 7 otherprop Objects 40 misc +829 stair handle stair handle 2 758 40 7 handle otherprop Objects n04047401 railing.n.01 30 railing +830 reflection reflection 2 64 11 8 picture picture Picture n04068976 reflection.n.05 6 picture +831 horizonal bar for exercise? exercise equipment 2 457 39 6 excercise equipment otherfurniture Furniture n04285146 sports_equipment.n.01 33 gym_equipment +832 shelving / other room shelving /otherroom 2 42 15 6 shelves shelves Furniture n04190052 shelf.n.01 31 shelving +833 seats seat 2 524 39 6 furniture otherfurniture Furniture n04161981 seat.n.03 34 seating +834 shower stall shower stall 2 40 7 otherprop Objects n04209613 shower_stall.n.01 40 misc +835 chairs chair 2 5 5 4 chair chair Chair chair chair chair 03001627 n03001627 chair.n.01 3 chair +836 stair /otherroom stair /otherroom 2 215 38 7 stairs otherstructure Objects stairs n04314914 step.n.04 16 stairs +837 kitchen stuff clutter 2 40 7 otherprop Objects 40 misc +838 unkown object object 2 40 7 otherprop Objects n00002684 object.n.01 39 objects +839 throne throne 2 5 5 4 chair chair Chair chair chair chair 03001627 n04429376 throne.n.01 3 chair +840 socket socket 2 40 7 otherprop Objects n04255163 socket.n.02 39 objects +841 bathroom art bathroom art 2 40 7 otherprop Objects n02743547 art.n.01 39 objects +842 night table night table 2 19 7 10 table table Table table table table 04379243 n04379243 table.n.02 5 table +843 display display 2 40 7 otherprop Objects n03211117 display.n.06 22 tv_monitor +844 tabletop tabletop 2 19 7 10 table table Table n04381860 tabletop.n.01 39 objects +845 thrash bin trash bin 2 307 40 7 bin otherprop Objects trash_bin 02747177 n02747177 ashcan.n.01 39 objects +846 l shape sofa l-shaped sofa 2 83 6 9 sofa sofa Sofa sofa sofa sofa 04256520 n04256520 sofa.n.01 10 sofa +847 cardboard box cardboard box 2 26 29 7 box box Objects 40 misc +848 wall borader wall board 1 408 38 7 board otherstructure Objects 40 misc +849 kitchen appliances kitchen appliance 1 40 7 otherprop Objects n03620052 kitchen_appliance.n.01 37 appliances +850 ceiling object object 1 40 7 otherprop Objects n00002684 object.n.01 39 objects +851 wall hanging decoration wall hanging decoration 1 40 7 otherprop Objects 40 misc +852 stand / small table stand/small table 1 19 7 10 table table Table table table table 04379243 n04379243 table.n.02 5 table +853 kitchen ceiling kitchen ceiling 1 4 22 3 ceiling ceiling Ceiling n02990373 ceiling.n.01 17 ceiling +854 floor matt floor mat 1 143 20 5 floor mat floor mat Floor n03727837 mat.n.01 2 floor +855 wall indent wall indent 1 40 7 otherprop Objects 40 misc +856 chairir chair 1 5 5 4 chair chair Chair chair chair chair 03001627 n03001627 chair.n.01 3 chair +857 window seat window seat 1 777 38 7 window seat otherstructure Objects bench bench 02828884 n04590021 window_seat.n.01 34 seating +858 bike bicycle 1 189 40 7 bicycle otherprop Objects bicycle 02834778 n02834778 bicycle.n.01 39 objects +859 towels (rolled) towel 1 135 27 7 towel towel Objects n04459362 towel.n.01 20 towel +860 wall \other oom wall /otherroom 1 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +861 shower pipe shower pipe 1 664 40 7 shower pipe otherprop Objects 40 misc +862 towel or curtain bar towel/curtain bar 1 51 38 7 bar otherstructure Objects 40 misc +863 shower glass shower glass 1 612 38 7 glass otherstructure Objects 40 misc +864 stone bench stone bench 1 204 39 6 bench otherfurniture Furniture bench bench 02828884 n02828884 bench.n.01 34 seating +865 window/other room window /otherroom 1 59 9 13 window window Window n04587648 window.n.01 9 window +866 back splash sink sink 1 24 34 7 sink sink Objects sink n04223580 sink.n.01 15 sink +867 iron board iron board 1 408 38 7 board otherstructure Objects 40 misc +868 computer equipment computer equipment 1 40 7 otherprop Objects 40 misc +869 shelf / cabinet shelf/cabinet 1 40 7 otherprop Objects 40 misc +870 stove door stove door 1 28 8 12 door door Wall door 40 misc +871 door inside door 1 28 8 12 door door Wall door n03221720 door.n.01 4 door +872 unknown stove accessory unknown stove accessory 1 40 7 otherprop Objects 40 misc +873 circular sofa circular sofa 1 83 6 9 sofa sofa Sofa sofa sofa sofa 04256520 n04256520 sofa.n.01 10 sofa +874 dustpan dustpan 1 40 7 otherprop Objects n03259009 dustpan.n.02 39 objects +875 bathroom door door 1 28 8 12 door door Wall door n03221720 door.n.01 4 door +876 paining painting 1 64 11 8 picture picture Picture n03876519 painting.n.01 39 objects +877 luggage luggage 1 783 40 7 luggage otherprop Objects n02774630 baggage.n.01 39 objects +878 dooframe doorframe 1 615 38 7 door frame otherstructure Objects n03222722 doorframe.n.01 4 door +879 outside wall wall 2 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +880 alarm control alarm control 1 40 7 otherprop Objects 40 misc +881 oil lamp oil lamp 1 144 35 7 lamp lamp Objects lamp lamp 03636649 n03636649 lamp.n.02 28 lighting +882 scaffolding scaffolding 1 40 7 otherprop Objects n04141712 scaffolding.n.01 39 objects +883 bed light bed light 1 62 38 7 light otherstructure Objects 40 misc +884 baluster baluster 1 453 38 7 banister otherstructure Objects n02783994 baluster.n.01 39 objects +885 leg rest leg rest 1 40 7 otherprop Objects 40 misc +886 ceiling / upstairs room ceiling /otheroom 1 40 7 otherprop Objects 40 misc +887 tv cabinet tv stand 1 291 39 6 tv stand otherfurniture Furniture tv_stand n03290653 entertainment_center.n.01 36 furniture +888 hole hole 1 40 7 otherprop Objects n09304750 hole.n.05 39 objects +889 ping pong table ping pong table 1 625 39 6 ping pong table otherfurniture Furniture table table table 04379243 n04379243 table.n.02 5 table +890 hutch hutch 1 40 7 otherprop Objects 40 misc +891 low shelf shelf 1 42 15 6 shelves shelves Furniture n04190052 shelf.n.01 31 shelving +892 foliage foliage 1 40 7 otherprop Objects 40 misc +893 windows sill window frame 1 59 9 13 window window Window n04589593 window_frame.n.01 9 window +894 toilet cistern toilet 1 124 33 7 toilet toilet Objects toilet toilet n04446276 toilet.n.01 18 toilet +895 stone circle stone circle 1 40 7 otherprop Objects 40 misc +896 bathroom sink sink 1 24 34 7 sink sink Objects sink n04223580 sink.n.01 15 sink +897 record player record player 1 220 40 7 machine otherprop Objects n04064401 record_player.n.01 39 objects +898 table cushion table cushion 1 40 7 otherprop Objects 40 misc +899 power outlet outlet 1 40 7 otherprop Objects n04548771 wall_socket.n.01 39 objects +900 machine machine 1 220 40 7 machine otherprop Objects n03699975 machine.n.01 39 objects +901 door post doorpost 1 40 7 otherprop Objects n03222857 doorjamb.n.01 39 objects +902 briefcase briefcase 1 617 40 7 briefcase otherprop Objects n02900705 briefcase.n.01 39 objects +903 wall of doors door 1 28 8 12 door door Wall door n03221720 door.n.01 4 door +904 showe wall shower wall 1 21 1 12 wall wall Wall n04208936 shower.n.01 23 shower +905 door stand door stand 1 50 39 6 stand otherfurniture Furniture 40 misc +906 energy box energy box 1 26 29 7 box box Objects 40 misc +907 balcony floor floor 1 11 2 5 floor floor Floor n03365592 floor.n.01 2 floor +908 bean bag beanbag 1 797 39 6 bean bag otherfurniture Furniture n02816656 beanbag.n.01 3 chair +909 kitchen extractor kitchen extractor 1 40 7 otherprop Objects 40 misc +910 toilet bin toilet bin 1 307 40 7 bin otherprop Objects 40 misc +911 wall in other room wall /otherroom 1 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +912 plumbing plumbing 1 40 7 otherprop Objects n03969041 plumbing.n.01 39 objects +913 moose head / sculpture / hunting trophy moose head/sculpture/hunting trophy 1 547 40 7 trophy otherprop Objects 40 misc +914 ceiling dome ceiling dome 1 40 7 otherprop Objects 40 misc +915 cabinet counter cabinet counter 1 7 12 6 counter counter Furniture 40 misc +916 cabinet \other room cabinet /otherroom 1 3 3 6 cabinet cabinet Furniture cabinet 02933112 n02933112 cabinet.n.01 7 cabinet +917 flowerbed flowerbed 1 157 4 1 bed bed Bed bed bed n03368352 flowerbed.n.01 39 objects +918 paingitn painting 1 64 11 8 picture picture Picture n03876519 painting.n.01 39 objects +919 bed headboard headboard 1 161 39 6 headboard otherfurniture Furniture n03502200 headboard.n.01 11 bed +920 antique clock antique clock 1 56 40 7 clock otherprop Objects clock 03046257 n03046257 clock.n.01 39 objects +921 rocks rock 1 40 7 otherprop Objects n09416076 rock.n.01 39 objects +922 shower caddy for soap etc. shower caddy 1 40 7 otherprop Objects 40 misc +923 window / frame window frame 1 59 9 13 window window Window n04589593 window_frame.n.01 9 window +924 media console media console 1 40 7 otherprop Objects 40 misc +925 cloths cloth 1 40 7 otherprop Objects n03309808 fabric.n.01 39 objects +926 vaccum cleaner vacuum cleaner 1 306 40 7 vacuum cleaner otherprop Objects n04517823 vacuum.n.04 37 appliances +927 door frame fireplace wall door frame 1 28 8 12 door door Wall door n03221720 door.n.01 4 door +928 fruit dish fruit dish 1 40 7 otherprop Objects 40 misc +929 risers for theater seating risers for theater seating 1 40 7 otherprop Objects 40 misc +930 interior bathroom wall wall 1 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +931 hall wall wall 1 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +932 drawer sink table drawer sink table 1 19 7 10 table table Table table table table 04379243 n04379243 table.n.02 5 table +933 keyboard piano keyboard piano 1 298 39 6 piano otherfurniture Furniture piano piano 03928116 n03928116 piano.n.01 39 objects +934 tree branch tree branch 1 40 7 otherprop Objects n13163803 limb.n.02 39 objects +935 tiled floor tiled floor 1 11 2 5 floor floor Floor 40 misc +936 ceiling bedroom ceiling bedroom 1 40 7 otherprop Objects 40 misc +937 chandellier chandelier 1 342 38 7 chandelier otherstructure Objects n03005285 chandelier.n.01 28 lighting +938 ceilin ceiling 1 4 22 3 ceiling ceiling Ceiling n02990373 ceiling.n.01 17 ceiling +939 hearst hearst 1 40 7 otherprop Objects n11037278 hearst.n.01 39 objects +940 random wall wall 1 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +941 condiments condiment 1 40 7 otherprop Objects 40 misc +942 shelf cubbies shelf cubby 1 40 7 otherprop Objects 40 misc +943 book display book display 1 40 7 otherprop Objects 40 misc +944 endtable end table 1 19 7 10 table table Table table table table 04379243 n04379243 table.n.02 5 table +945 antique telephone antique telephone 1 32 40 7 telephone otherprop Objects telephone 04401088 n04401088 telephone.n.01 39 objects +946 clothes rack clothes rack 1 40 7 otherprop Objects 40 misc +947 doore door 1 28 8 12 door door Wall door n03221720 door.n.01 4 door +948 heater piping heater piping 1 40 7 otherprop Objects 40 misc +949 tissue box stand tissue box stand 1 50 39 6 stand otherfurniture Furniture 40 misc +950 utencils utensil 1 267 40 7 utensil otherprop Objects n04516672 utensil.n.01 39 objects +951 foodtray food tray 1 179 40 7 tray otherprop Objects n04476259 tray.n.01 39 objects +952 shelves /otherroom shelves /otherroom 1 42 15 6 shelves shelves Furniture 40 misc +953 cailing ceiling 1 4 22 3 ceiling ceiling Ceiling n02990373 ceiling.n.01 17 ceiling +954 cluttered chair chair 1 5 5 4 chair chair Chair chair chair chair 03001627 n03001627 chair.n.01 3 chair +955 countertop object object 1 40 7 otherprop Objects n00002684 object.n.01 39 objects +956 art frames art frame 1 40 7 otherprop Objects 40 misc +957 soap bottle soap bottle 1 2 40 7 bottle otherprop Objects bottle bottle 02876657 n02876657 bottle.n.01 39 objects +958 small wall wall 1 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +959 watch watch 1 384 40 7 watch otherprop Objects n04555897 watch.n.01 39 objects +960 ceil rail 1 497 38 7 railing otherstructure Objects n04047401 railing.n.01 30 railing +961 fuse box fuse box 1 26 29 7 box box Objects 40 misc +962 painting board painting 1 64 11 8 picture picture Picture n03876519 painting.n.01 39 objects +963 door locker handle door locker handle 1 758 40 7 handle otherprop Objects 40 misc +964 box with clutter box 1 26 29 7 box box Objects n02883344 box.n.01 39 objects +965 knive holder knife holder 1 40 7 otherprop Objects 40 misc +966 computer mouse computer mouse 1 103 40 7 mouse otherprop Objects n03793489 mouse.n.04 39 objects +967 mirror / other room mirror /otherroom 1 122 19 7 mirror mirror Objects n03773035 mirror.n.01 21 mirror +968 door back side door 1 28 8 12 door door Wall door n03221720 door.n.01 4 door +969 ceiling bath ceiling bath 1 40 7 otherprop Objects 40 misc +970 chandelier? chandelier 1 342 38 7 chandelier otherstructure Objects n03005285 chandelier.n.01 28 lighting +971 table tray table tray 1 179 40 7 tray otherprop Objects 40 misc +972 aquarium aquarium 1 263 40 7 vessel otherprop Objects n02732072 aquarium.n.01 40 misc +973 wheelbarrow wheelbarrow 1 305 40 7 cart otherprop Objects n02797295 barrow.n.03 39 objects +974 curtain rail curtain rail 1 40 7 otherprop Objects 40 misc +975 rods / table rods/table 1 40 7 otherprop Objects 40 misc +976 counter /otherroom counter /otherroom 1 7 12 6 counter counter Furniture table table table 04379243 n03116530 counter.n.01 26 counter +977 gable gable 1 21 1 12 wall wall Wall n03409393 gable.n.01 1 wall +978 bothroom wall wall 1 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +979 balustrade balustrade 1 453 38 7 banister otherstructure Objects n02788148 bannister.n.02 30 railing +980 bathroom bathroom 1 40 7 otherprop Objects toilet toilet 40 misc +981 three three 1 40 7 otherprop Objects 40 misc +982 handcloth hand cloth 1 40 7 otherprop Objects 40 misc +983 wall curved wall 1 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +984 teapoy coffee table 1 356 39 6 coffee table otherfurniture Furniture table table table 04379243 n03063968 coffee_table.n.01 5 table +985 table with clutter table 1 19 7 10 table table Table table table table 04379243 n04379243 table.n.02 5 table +986 back door door 1 28 8 12 door door Wall door n03221720 door.n.01 4 door +987 rocky ground rocky ground 1 40 7 otherprop Objects 40 misc +988 doormat doormat 2 143 20 5 floor mat floor mat Floor n03223299 doormat.n.02 2 floor +989 backrest backrest 1 5 5 4 chair chair Chair n02767433 back.n.08 39 objects +990 cabinet with desk cabinet 1 3 3 6 cabinet cabinet Furniture cabinet 02933112 n02933112 cabinet.n.01 7 cabinet +991 faucet handles faucet 1 9 40 7 faucet otherprop Objects faucet 03325088 n03325088 faucet.n.01 39 objects +992 floor trim floor trim 1 868 38 7 floor trim otherstructure Objects 40 misc +993 toilet sliding door toilet sliding door 1 28 8 12 door door Wall door 40 misc +994 gym stuff gym equipment 1 457 39 6 excercise equipment otherfurniture Furniture n04285146 sports_equipment.n.01 33 gym_equipment +995 clothes container clothes container 1 140 40 7 container otherprop Objects 40 misc +996 basketball hoop basketball hoop 1 162 40 7 basketball hoop otherprop Objects n02802215 basket.n.03 33 gym_equipment +997 cooktop stovetop 1 40 7 otherprop Objects 40 misc +998 both tub bathtub 1 136 36 7 bathtub bathtub Objects bathtub bathtub tub 02808440 n02808440 bathtub.n.01 25 bathtub +999 fireplace heart fireplace 1 372 38 7 fireplace otherstructure Objects n03346455 fireplace.n.01 27 fireplace +1000 spice racks spice rack 1 241 38 7 spice rack otherstructure Objects n04275175 spice_rack.n.01 31 shelving +1001 wall fireplace wall 1 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +1002 cabinet /w cluttered art cabinet /w cluttered art 1 40 7 otherprop Objects 40 misc +1003 inside wall wall 1 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +1004 transformer transformer 1 40 7 otherprop Objects n04471315 transformer.n.01 39 objects +1005 wall light wall lamp 1 144 35 7 lamp lamp Objects lamp lamp 03636649 n03636649 lamp.n.02 28 lighting +1006 bathroom shelf bathroom shelf 1 42 15 6 shelves shelves Furniture n04190052 shelf.n.01 31 shelving +1007 glass french door door 1 28 8 12 door door Wall door n03221720 door.n.01 4 door +1008 lamp / other room lamp /otherroom 1 144 35 7 lamp lamp Objects lamp lamp 03636649 n03636649 lamp.n.02 28 lighting +1009 picutre picture 1 64 11 8 picture picture Picture n03931044 picture.n.01 6 picture +1010 gift gift 1 40 7 otherprop Objects 40 misc +1011 mirror door mirror door 1 28 8 12 door door Wall door 40 misc +1012 deorative object decoration 1 40 7 otherprop Objects n03169390 decoration.n.01 39 objects +1013 ceiling door ceiling door 1 28 8 12 door door Wall door 40 misc +1014 stack of papers stack of papers 1 483 40 7 papers otherprop Objects 40 misc +1015 holy cross holy cross 1 40 7 otherprop Objects 40 misc +1016 door window door window 1 59 9 13 window window Window 40 misc +1017 computer screen monitor 1 49 40 7 monitor otherprop Objects monitor monitor tv or monitor 03211117 n03782190 monitor.n.04 22 tv_monitor +1018 while bottles bottle 1 2 40 7 bottle otherprop Objects bottle bottle 02876657 n02876657 bottle.n.01 39 objects +1019 arcade game arcade game 1 40 7 otherprop Objects 40 misc +1020 unknown - probably part of trellis -- maybe blinds unknown - probably part of trellis -- maybe blinds 1 80 13 13 blinds blinds Window 40 misc +1021 compound wall compound wall 1 21 1 12 wall wall Wall 40 misc +1022 lamps lamp 1 144 35 7 lamp lamp Objects lamp lamp 03636649 n03636649 lamp.n.02 28 lighting +1023 plug plug 1 40 7 otherprop Objects 40 misc +1024 round cushion round cushion 1 40 7 otherprop Objects 40 misc +1025 ceiling frame molding 1 38 7 otherstructure Objects n02800354 baseboard.n.01 1 wall +1026 magazine magazine 1 71 40 7 magazine otherprop Objects n06595351 magazine.n.01 39 objects +1027 stair case staircase 1 215 38 7 stairs otherstructure Objects n04298308 stairway.n.01 16 stairs +1028 rolling pin rolling pin 1 267 40 7 utensil otherprop Objects n04103206 rolling_pin.n.01 39 objects +1029 shower knob shower knob 1 651 40 7 shower knob otherprop Objects 40 misc +1030 wall statue wall statue 1 40 7 otherprop Objects 40 misc +1031 wal wall 1 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +1032 sink / basin sink/basin 1 40 7 otherprop Objects 40 misc +1033 celling borader ceiling boarder 1 40 7 otherprop Objects 40 misc +1034 clothes bag clothes bag 1 55 37 7 bag bag Objects 40 misc +1035 perfumes perfume 1 655 40 7 perfume otherprop Objects n03916031 perfume.n.02 39 objects +1036 heat heat 1 40 7 otherprop Objects n03509025 heating_system.n.01 39 objects +1037 kitchen counter support leg kitchen counter support 1 40 7 otherprop Objects 40 misc +1038 window frame window frame 1 59 9 13 window window Window n04589593 window_frame.n.01 9 window +1039 glassdoor door 1 28 8 12 door door Wall door n03221720 door.n.01 4 door +1040 window closet door window closet door 1 28 8 12 door door Wall door 40 misc +1041 sticks stick 1 529 40 7 stick otherprop Objects 40 misc +1042 photo frame picture frame 1 64 11 8 picture picture Picture n03931765 picture_frame.n.01 6 picture +1043 water pump water pump 1 40 7 otherprop Objects n04561965 water_pump.n.01 39 objects +1044 shower doorframe shower door frame 1 40 7 otherprop Objects n04208936 shower.n.01 23 shower +1045 stairs behind stairs 1 215 38 7 stairs otherstructure Objects stairs n04314914 step.n.04 16 stairs +1046 flood floor 1 11 2 5 floor floor Floor n03365592 floor.n.01 2 floor +1047 columned perimeter columned perimeter 1 40 7 otherprop Objects 40 misc +1048 shrine shrine 1 40 7 otherprop Objects n04210390 shrine.n.01 40 misc +1049 papers paper 1 15 26 7 paper paper Objects n14974264 paper.n.01 39 objects +1050 fireplace /w art fireplace 1 372 38 7 fireplace otherstructure Objects n03346455 fireplace.n.01 27 fireplace +1051 canvas stand canvas stand 1 50 39 6 stand otherfurniture Furniture 40 misc +1052 art / man statue art/man statue 1 40 7 otherprop Objects 40 misc +1053 bath towel bath towel 1 135 27 7 towel towel Objects n02808304 bath_towel.n.01 20 towel +1054 cradenza credenza 1 7 12 6 counter counter Furniture n03129753 credenza.n.01 36 furniture +1055 doorr frame door frame 1 28 8 12 door door Wall door n03221720 door.n.01 4 door +1056 column statue column statue 1 40 7 otherprop Objects 40 misc +1057 water basin water basin 1 40 7 otherprop Objects 40 misc +1058 moulidng molding 1 38 7 otherstructure Objects n02800354 baseboard.n.01 1 wall +1059 tablet computer tablet computer 1 46 40 7 computer otherprop Objects 40 misc +1060 artwork artwork 1 64 11 8 picture picture Picture n03931044 picture.n.01 6 picture +1061 towel bar shelf towel bar shelf 1 40 7 otherprop Objects 40 misc +1062 floor carpet carpet 1 130 40 7 rug floor mat Objects n04118021 rug.n.01 2 floor +1063 wine cabinet wine cabinet 1 3 3 6 cabinet cabinet Furniture cabinet 02933112 n02933112 cabinet.n.01 7 cabinet +1064 shower rail shower rail 1 40 7 otherprop Objects 40 misc +1065 skirting board skirting board 1 408 38 7 board otherstructure Objects n02800354 baseboard.n.01 1 wall +1066 playpen playpen 1 815 39 6 playpen otherfurniture Furniture n03964495 playpen.n.01 40 misc +1067 room door door 1 28 8 12 door door Wall door n03221720 door.n.01 4 door +1068 doorgrame doorframe 1 615 38 7 door frame otherstructure Objects n03222722 doorframe.n.01 4 door +1069 makeup makeup 1 40 7 otherprop Objects n03714235 makeup.n.01 39 objects +1070 plant / art plant/art 1 40 7 otherprop Objects 40 misc +1071 sauna seating seat 1 524 39 6 furniture otherfurniture Furniture n04161981 seat.n.03 34 seating +1072 bot bot 1 40 7 otherprop Objects n02311879 bot.n.01 39 objects +1073 handrail / other room handrail /otherroom 1 453 38 7 banister otherstructure Objects n02788148 bannister.n.02 30 railing +1074 rocking horse rocking horse 1 389 40 7 toy otherprop Objects n03523633 hobby.n.02 39 objects +1075 kitchen lower cabinet kitchen lower cabinet 1 3 3 6 cabinet cabinet Furniture cabinet 02933112 n02933112 cabinet.n.01 7 cabinet +1076 floor /other room floor /otherroom 1 11 2 5 floor floor Floor n03365592 floor.n.01 2 floor +1077 chair stand chair stand 1 50 39 6 stand otherfurniture Furniture 40 misc +1078 wall pack wall pack 1 40 7 otherprop Objects 40 misc +1079 towel box towel box 1 26 29 7 box box Objects 40 misc +1080 vessel sink vessel sink 1 24 34 7 sink sink Objects sink 40 misc +1081 can can 1 329 40 7 can otherprop Objects can 02946921 n02946921 can.n.01 39 objects +1082 ceiling lattice ceiling 1 4 22 3 ceiling ceiling Ceiling n02990373 ceiling.n.01 17 ceiling +1083 backslash backsplash 1 40 7 otherprop Objects 40 misc +1084 ire place fireplace 1 372 38 7 fireplace otherstructure Objects n03346455 fireplace.n.01 27 fireplace +1085 divider partition 1 21 1 12 wall wall Wall n03894379 partition.n.01 40 misc +1086 pathway pathway 1 40 7 otherprop Objects n03899533 pathway.n.02 40 misc +1087 laundry laundry 1 40 7 otherprop Objects n03648219 laundry.n.01 38 clothes +1088 tablecloth tablecloth 1 292 40 7 tablecloth otherprop Objects n04380143 tablecloth.n.01 39 objects +1089 water dispenser water dispenser 1 507 40 7 water dispenser otherprop Objects n03210683 dispenser.n.01 39 objects +1090 ;photo photo 1 508 40 7 photo otherprop Objects n03925226 photograph.n.01 6 picture +1091 door frame wall entry door frame 1 28 8 12 door door Wall door n03221720 door.n.01 4 door +1092 plastic tarp tarp 1 40 7 otherprop Objects n04395024 tarpaulin.n.01 39 objects +1093 clothing stand clothing stand 1 50 39 6 stand otherfurniture Furniture 40 misc +1094 bathtab bathtub 1 136 36 7 bathtub bathtub Objects bathtub bathtub tub 02808440 n02808440 bathtub.n.01 25 bathtub +1095 apron apron 1 40 7 otherprop Objects 40 misc +1096 ceiling bedroom entry ceiling 1 4 22 3 ceiling ceiling Ceiling n02990373 ceiling.n.01 17 ceiling +1097 ashtray ashtray 1 377 40 7 ashtray otherprop Objects n02747802 ashtray.n.01 39 objects +1098 control panel control panel 1 408 38 7 board otherstructure Objects n03098140 control_panel.n.01 39 objects +1099 ironing board ironing board 1 313 39 6 ironing board otherfurniture Furniture n03586090 ironing_board.n.01 39 objects +1100 exhaust pipe exhaust pipe 1 41 40 7 pipe otherprop Objects n03303510 exhaust_pipe.n.01 40 misc +1101 rack of theater electronics rack 1 50 39 6 stand otherfurniture Furniture n04038440 rack.n.05 31 shelving +1102 canvas canvas 1 559 40 7 sheet otherprop Objects 40 misc +1103 alarm clock alarm clock 1 156 40 7 alarm clock otherprop Objects clock 03046257 n02694662 alarm_clock.n.01 39 objects +1104 ceiling lower ceiling lower 1 40 7 otherprop Objects 40 misc +1105 yarn machine yarn machine 1 220 40 7 machine otherprop Objects 40 misc +1106 tarrace door terrace door 1 28 8 12 door door Wall door 40 misc +1107 stand/table stand/table 1 40 7 otherprop Objects 40 misc +1108 statue base statue 1 294 40 7 sculpture otherprop Objects n04306847 statue.n.01 39 objects +1109 swing swing 1 389 40 7 toy otherprop Objects n04371774 swing.n.02 39 objects +1110 extractor fan extractor fan 1 74 40 7 fan otherprop Objects 40 misc +1111 crib crib 1 485 39 6 crib otherfurniture Furniture 40 misc +1112 hallway wall wall 1 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +1113 side wall 2 wall 1 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +1114 side wall 5 wall 1 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +1115 mens urinal urinal 1 40 7 otherprop Objects n04515991 urinal.n.01 39 objects +1116 stacked chairs stacked chair 1 5 5 4 chair chair Chair chair chair chair 03001627 n03001627 chair.n.01 3 chair +1117 patio floor patio floor 1 11 2 5 floor floor Floor 40 misc +1118 stair steps stair step 1 40 7 otherprop Objects 40 misc +1119 electrical controller electrical controller 1 40 7 otherprop Objects 40 misc +1120 kitchencounter kitchen counter 1 7 12 6 counter counter Furniture table table table 04379243 n03116530 counter.n.01 26 counter +1121 rotunda railing rotunda railing 1 497 38 7 railing otherstructure Objects 40 misc +1122 lamp table lamp table 1 19 7 10 table table Table table table table 04379243 n04379243 table.n.02 5 table +1123 cabinet top cabinet 1 3 3 6 cabinet cabinet Furniture cabinet 02933112 n02933112 cabinet.n.01 7 cabinet +1124 tray with bottles of soap and lotion tray 1 179 40 7 tray otherprop Objects n04476259 tray.n.01 39 objects +1125 speakers speaker 1 54 40 7 speaker otherprop Objects speaker 03691459 n03691459 loudspeaker.n.01 39 objects +1126 fancy cabinet cabinet 1 3 3 6 cabinet cabinet Furniture cabinet 02933112 n02933112 cabinet.n.01 7 cabinet +1127 air vent air vent 1 25 38 7 air vent otherstructure Objects n04526241 vent.n.01 40 misc +1128 sofa couch couch 1 83 6 9 sofa sofa Sofa sofa sofa sofa 04256520 n04256520 sofa.n.01 10 sofa +1129 channel channel 1 40 7 otherprop Objects 40 misc +1130 wall sign wall sign 1 208 40 7 sign otherprop Objects 40 misc +1131 bench back bench 1 204 39 6 bench otherfurniture Furniture bench bench 02828884 n02828884 bench.n.01 34 seating +1132 baby changing station baby changing station 1 40 7 otherprop Objects 40 misc +1133 paip pip 1 286 40 7 fruit otherprop Objects n11685091 pip.n.03 39 objects +1134 bed arm pillows pillow 1 119 18 7 pillow pillow Objects pillow 03938244 n03938244 pillow.n.01 8 cushion +1135 shower tap shower tap 1 132 40 7 shower cap otherprop Objects 40 misc +1136 axe axe 1 40 7 otherprop Objects n02764044 ax.n.01 39 objects +1137 table shelf shelf 1 42 15 6 shelves shelves Furniture n04190052 shelf.n.01 31 shelving +1138 washcloth washcloth 1 40 7 otherprop Objects n04554523 washcloth.n.01 39 objects +1139 coffee mug coffee mug 1 263 40 7 vessel otherprop Objects cup or mug 03797390 n03063599 coffee_mug.n.01 39 objects +1140 iron panel panel 1 559 40 7 sheet otherprop Objects n03882058 panel.n.01 35 board_panel +1141 door or window frame door/window frame 1 40 7 otherprop Objects 40 misc +1142 coffe table coffee table 1 356 39 6 coffee table otherfurniture Furniture table table table 04379243 n03063968 coffee_table.n.01 5 table +1143 dice dice 1 40 7 otherprop Objects n03191029 die.n.01 39 objects +1144 title title 1 40 7 otherprop Objects 40 misc +1145 toilet seat toilet 1 124 33 7 toilet toilet Objects toilet toilet n04446276 toilet.n.01 18 toilet +1146 coffee table leg coffee table 1 356 39 6 coffee table otherfurniture Furniture table table table 04379243 n03063968 coffee_table.n.01 5 table +1147 soft chair soft chair 1 5 5 4 chair chair Chair chair chair chair 03001627 n03001627 chair.n.01 3 chair +1148 icebox icebox 1 17 24 6 refridgerator refridgerator Furniture n04070727 refrigerator.n.01 37 appliances +1149 showerfloor shower floor 1 11 2 5 floor floor Floor n04208936 shower.n.01 23 shower +1150 floor outside floor /outside 1 11 2 5 floor floor Floor n03365592 floor.n.01 2 floor +1151 framing framing 1 40 7 otherprop Objects n03390983 frame.n.10 40 misc +1152 wall patch wall 1 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +1153 decorative bowl decorative bowl 1 826 40 7 decorative bowl otherprop Objects bowl bowl 02880940 n02880940 bowl.n.03 39 objects +1154 dog bed dog bed 1 858 39 6 dog bed otherfurniture Furniture bed bed bed 02818832 n02818832 bed.n.01 11 bed +1155 pool sticks pool stick 1 529 40 7 stick otherprop Objects n03145522 cue.n.04 39 objects +1156 exericse equipment exercise equipment 1 457 39 6 excercise equipment otherfurniture Furniture n04285146 sports_equipment.n.01 33 gym_equipment +1157 remove floor behind floor /otherroom 1 11 2 5 floor floor Floor n03365592 floor.n.01 2 floor +1158 doorhandle door handle 1 758 40 7 handle otherprop Objects 40 misc +1159 dishrag dishrag 1 40 7 otherprop Objects n03207743 dishrag.n.01 39 objects +1160 behind behind 1 40 7 otherprop Objects 40 misc +1161 drinking fountain drinking fountain 1 40 7 otherprop Objects n03241335 drinking_fountain.n.01 40 misc +1162 bureau bureau 1 524 39 6 furniture otherfurniture Furniture dresser dresser n03015254 chest_of_drawers.n.01 13 chest_of_drawers +1163 night stand object object 1 40 7 otherprop Objects n00002684 object.n.01 39 objects +1164 tub bathtub 1 136 36 7 bathtub bathtub Objects bathtub bathtub tub 02808440 n02808440 bathtub.n.01 25 bathtub +1165 parapet parapet 1 40 7 otherprop Objects 40 misc +1166 attic door attic door 1 28 8 12 door door Wall door 40 misc +1167 wall object object 1 40 7 otherprop Objects n00002684 object.n.01 39 objects +1168 closet door knob closet door knob 1 652 40 7 knob otherprop Objects 40 misc +1169 bathroom accessory bathroom accessory 1 40 7 otherprop Objects n02671780 accessory.n.01 38 clothes +1170 teddy teddy bear 1 389 40 7 toy otherprop Objects n04399382 teddy.n.01 39 objects +1171 wall toilet paper wall toilet paper 1 15 26 7 paper paper Objects 40 misc +1172 coffee machine coffee machine 1 234 40 7 coffee machine otherprop Objects n03063338 coffee_maker.n.01 37 appliances +1173 excercise equipment / other room exercise equipment /otherroom 1 457 39 6 excercise equipment otherfurniture Furniture 40 misc +1174 storage cabinet storage cabinet 1 3 3 6 cabinet cabinet Furniture cabinet 02933112 n02933112 cabinet.n.01 7 cabinet +1175 toilet stall door toilet stall door 1 28 8 12 door door Wall door 40 misc +1176 bin of posters bin 1 307 40 7 bin otherprop Objects n02839910 bin.n.01 39 objects +1177 photos picture 1 64 11 8 picture picture Picture n03931044 picture.n.01 6 picture +1178 push broom broom 1 328 40 7 broom otherprop Objects n02906734 broom.n.01 39 objects +1179 art stand art stand 1 50 39 6 stand otherfurniture Furniture 40 misc +1180 shelf /w art shelf /w art 1 40 7 otherprop Objects 40 misc +1181 spiderman spiderman statue 1 40 7 otherprop Objects 40 misc +1182 mount mount 1 40 7 otherprop Objects 40 misc +1183 ac unit air conditioning 1 40 7 otherprop Objects n02686379 air_conditioner.n.01 39 objects +1184 newspaper newspaper 1 873 40 7 newspapers otherprop Objects 40 misc +1185 towel basket towel basket 1 39 40 7 basket otherprop Objects basket 02801938 n02801938 basket.n.01 39 objects +1186 base rail base rail 1 40 7 otherprop Objects 40 misc +1187 show pack show pack 1 40 7 otherprop Objects 40 misc +1188 coutertop countertop 1 7 12 6 counter counter Furniture n03118245 countertop.n.01 26 counter +1189 stair stepper stair stepper 1 40 7 otherprop Objects 40 misc +1190 car batteries car battery 1 40 7 otherprop Objects n02961225 car_battery.n.01 39 objects +1191 pad pad 1 40 7 otherprop Objects n03195485 diggings.n.02 40 misc +1192 old fireplace fireplace 1 372 38 7 fireplace otherstructure Objects n03346455 fireplace.n.01 27 fireplace +1193 desk lamp desk lamp 1 144 35 7 lamp lamp Objects lamp lamp 03636649 n03636649 lamp.n.02 28 lighting +1194 easychair easy chair 1 5 5 4 chair chair Chair chair chair chair 03001627 n03262932 easy_chair.n.01 3 chair +1195 wees table 1 19 7 10 table table Table table table table 04379243 n04379243 table.n.02 5 table +1196 tabletop box tabletop box 1 26 29 7 box box Objects 40 misc +1197 canopy canopy 1 40 7 otherprop Objects 40 misc +1198 wall behind door wall 1 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +1199 cabinet / other room cabinet /otherroom 1 3 3 6 cabinet cabinet Furniture cabinet 02933112 n02933112 cabinet.n.01 7 cabinet +1200 objects / other room objects /otherroom 1 40 7 otherprop Objects n00002684 object.n.01 39 objects +1201 bar soap bar soap 1 133 40 7 soap otherprop Objects 40 misc +1202 column /otherroom column /otherroom 1 94 38 7 column otherstructure Objects n03074380 column.n.06 40 misc +1203 outer wall wall 1 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +1204 wall control wall control 1 40 7 otherprop Objects 40 misc +1205 desktop computer tower 1 46 40 7 computer otherprop Objects n03082979 computer.n.01 39 objects +1206 fire screen fire screen 1 40 7 otherprop Objects n03347037 fire_screen.n.01 39 objects +1207 fireplace counter mantel 1 58 38 7 mantel otherstructure Objects mantel n03719343 mantel.n.01 27 fireplace +1208 bread bread 1 246 40 7 bread otherprop Objects n07679356 bread.n.01 40 misc +1209 piano bench piano bench 1 460 39 6 piano bench otherfurniture Furniture bench bench 02828884 n02828884 bench.n.01 34 seating +1210 window shutter window shutter 1 40 7 otherprop Objects n04211356 shutter.n.02 39 objects +1211 draw draw 1 40 7 otherprop Objects n09269882 draw.n.01 39 objects +1212 flower wage flower vase 1 78 40 7 vase otherprop Objects vase jar 03593526 n04522168 vase.n.01 39 objects +1213 ceiling/west wall ceiling/west wall 1 21 1 12 wall wall Wall 40 misc +1214 decorative quilt decorative quilt 1 575 40 7 quilt otherprop Objects 40 misc +1215 knife knife 1 259 40 7 knife otherprop Objects knife 03624134 n03624134 knife.n.02 39 objects +1216 projector opening projector opening 1 40 7 otherprop Objects 40 misc +1217 boader boarder 1 40 7 otherprop Objects 40 misc +1218 lights / deco lights/deco 1 40 7 otherprop Objects 40 misc +1219 tv3 wall tv 1 40 7 otherprop Objects 40 misc +1220 desk clutter desk clutter 1 40 7 otherprop Objects 40 misc +1221 curatain curtain 1 89 16 13 curtain curtain Window curtain n03151077 curtain.n.01 12 curtain +1222 shoe shelves shelving 1 42 15 6 shelves shelves Furniture n04190052 shelf.n.01 31 shelving +1223 mixer mixer 1 40 7 otherprop Objects 40 misc +1224 ceiling fixture ceiling fixture 1 40 7 otherprop Objects 40 misc +1225 scuplture sculpture 1 294 40 7 sculpture otherprop Objects n04157320 sculpture.n.01 39 objects +1226 countertop /otherroom countertop /otherroom 1 7 12 6 counter counter Furniture n03118245 countertop.n.01 26 counter +1227 work bench workbench 1 204 39 6 bench otherfurniture Furniture bench table 04379243 n04600486 workbench.n.01 5 table +1228 wall desk wall desk 1 36 14 10 desk desk Table desk desk 40 misc +1229 shelf of cloth shelf 1 42 15 6 shelves shelves Furniture n04190052 shelf.n.01 31 shelving +1230 telescope telescope 1 467 40 7 telescope otherprop Objects n04403638 telescope.n.01 39 objects +1231 shower hoses shower hose 1 669 40 7 shower hose otherprop Objects 40 misc +1232 step; step 1 38 7 otherstructure Objects n04314914 step.n.04 16 stairs +1233 bathtub platform bathtub platform 1 40 7 otherprop Objects 40 misc +1234 sauna seats seat 1 524 39 6 furniture otherfurniture Furniture n04161981 seat.n.03 34 seating +1235 bucker bucket 1 427 40 7 bucket otherprop Objects n02909870 bucket.n.01 39 objects +1236 ac air conditioner 1 79 38 7 air conditioner otherstructure Objects n02686379 air_conditioner.n.01 39 objects +1237 unknown / remove unknown/remove 1 40 7 otherprop Objects 40 misc +1238 art / statue art/statue 1 40 7 otherprop Objects 40 misc +1239 dinner table dinner table 1 19 7 10 table table Table table table table 04379243 n04379243 table.n.02 5 table +1240 storage storage 1 n03744276 memory.n.04 39 objects +1241 unknown kitchen stuff unknown kitchen stuff 1 40 7 otherprop Objects 40 misc +1242 alamari wardrobe 1 772 39 6 wardrobe otherfurniture Furniture wardrobe n04550184 wardrobe.n.01 36 furniture +1243 cement drum cement drum 1 145 40 7 drum otherprop Objects 40 misc +1244 kitchen cupboard kitchen cabinet 1 3 3 6 cabinet cabinet Furniture n02933112 cabinet.n.01 7 cabinet +1245 platter platter 1 129 40 7 platter otherprop Objects 40 misc +1246 large chunk of art chunk of art 1 40 7 otherprop Objects 40 misc +1247 dushbin dustbin 1 307 40 7 bin otherprop Objects trash_bin 02747177 n02747177 ashcan.n.01 39 objects +1248 couch pillows pillow 1 119 18 7 pillow pillow Objects pillow 03938244 n03938244 pillow.n.01 8 cushion +1249 ceiling /other room ceiling /otherroom 1 4 22 3 ceiling ceiling Ceiling n02990373 ceiling.n.01 17 ceiling +1250 object on cutting board object 1 40 7 otherprop Objects n00002684 object.n.01 39 objects +1251 show wall wall 1 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +1252 robes robe 1 40 7 otherprop Objects n04097866 robe.n.01 38 clothes +1253 cusion cushion 1 119 18 7 pillow pillow Objects n03151500 cushion.n.03 8 cushion +1254 column /outside column /outside 1 94 38 7 column otherstructure Objects n03074380 column.n.06 40 misc +1255 hanging clothes hanging clothes 1 141 21 7 clothes clothes Objects n02728440 apparel.n.01 38 clothes +1256 newspaper basket newspaper basket 1 39 40 7 basket otherprop Objects basket 02801938 n02801938 basket.n.01 39 objects +1257 towels in a bowl towel 1 135 27 7 towel towel Objects n04459362 towel.n.01 20 towel +1258 detached door door 1 28 8 12 door door Wall door n03221720 door.n.01 4 door +1259 big door door 1 28 8 12 door door Wall door n03221720 door.n.01 4 door +1260 wall molding molding 1 38 7 otherstructure Objects n02800354 baseboard.n.01 1 wall +1261 bed / other room bed /otherroom 1 157 4 1 bed bed Bed bed bed bed 02818832 n02818832 bed.n.01 11 bed +1262 window /w pictures window /w pictures 1 40 7 otherprop Objects 40 misc +1263 ceiling corridor ceiling corridor 1 40 7 otherprop Objects 40 misc +1264 rotunda base rotunda 1 40 7 otherprop Objects 40 misc +1265 fire extinguisher' fire extinguisher 1 10 40 7 fire extinguisher otherprop Objects n03345837 fire_extinguisher.n.01 39 objects +1266 unicycle unicycle 1 40 7 otherprop Objects n04509417 unicycle.n.01 39 objects +1267 ceiling under balcony ceiling /otherroom 1 4 22 3 ceiling ceiling Ceiling n02990373 ceiling.n.01 17 ceiling +1268 laundary machines laundry machine 1 220 40 7 machine otherprop Objects 40 misc +1269 nightstand /reflection nightstand /reflection 1 158 32 6 night stand night stand Furniture night_stand night_stand n03015254 chest_of_drawers.n.01 13 chest_of_drawers +1270 pile of magazines pile of magazines 1 40 7 otherprop Objects 40 misc +1271 wall of statue inset wall of statue inset 1 40 7 otherprop Objects 40 misc +1272 cosmetic cosmetic 1 40 7 otherprop Objects n03113152 cosmetic.n.01 39 objects +1273 unknown item object 1 40 7 otherprop Objects n00002684 object.n.01 39 objects +1274 side steps 4 step 1 38 7 otherstructure Objects n04314914 step.n.04 16 stairs +1275 kitchen table kitchen table 1 19 7 10 table table Table table table table 04379243 n03620967 kitchen_table.n.01 5 table +1276 round footstool round footstool 1 40 7 otherprop Objects 40 misc +1277 firewood holder firewood holder 1 40 7 otherprop Objects 40 misc +1278 flag flag 1 405 40 7 flag otherprop Objects 40 misc +1279 window frame and shelves window frame and shelves 1 42 15 6 shelves shelves Furniture 40 misc +1280 towel ring towel ring 1 40 7 otherprop Objects n04460038 towel_ring.n.01 39 objects +1281 window frame /reflection window frame /reflection 1 477 38 7 window frame otherstructure Objects n04589593 window_frame.n.01 9 window +1282 basket of something basket of something 1 40 7 otherprop Objects 40 misc +1283 towel rod towel rod 1 134 38 7 towel rod otherstructure Objects 40 misc +1284 window /reflection window /reflection 1 59 9 13 window window Window n04587648 window.n.01 9 window +1285 paper storage paper storage 1 40 7 otherprop Objects 40 misc +1286 t.v tv 1 172 25 11 television television TV tv or monitor 03211117 n03211117 display.n.06 22 tv_monitor +1287 kitchen bell kitchen bell 1 40 7 otherprop Objects 40 misc +1288 ceiling from another room ceiling /otherroom 1 4 22 3 ceiling ceiling Ceiling n02990373 ceiling.n.01 17 ceiling +1289 throw blanket throw blanket 1 312 40 7 blanket otherprop Objects 40 misc +1290 wall post wall post 1 40 7 otherprop Objects 40 misc +1291 sink pipe sink pipe 1 41 40 7 pipe otherprop Objects 40 misc +1292 workstation workstation 1 46 40 7 computer otherprop Objects n04603399 workstation.n.01 39 objects +1293 room 1 wall wall 1 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +1294 storage space storage space 1 645 38 7 storage space otherstructure Objects n04328946 storage_space.n.01 40 misc +1295 mattress bed 1 157 4 1 bed bed Bed bed bed bed 02818832 n02818832 bed.n.01 11 bed +1296 kitchen handle kitchen handle 1 758 40 7 handle otherprop Objects n03485997 handle.n.01 39 objects +1297 bed side table nightstand 1 158 32 6 night stand night stand Furniture night_stand night_stand n03015254 chest_of_drawers.n.01 13 chest_of_drawers +1298 lookout lookout 1 40 7 otherprop Objects n03688943 lookout.n.03 40 misc +1299 bag of sand bag of sand 1 40 7 otherprop Objects 40 misc +1300 walll wall 1 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +1301 kitchen countertop object' object 1 40 7 otherprop Objects n00002684 object.n.01 39 objects +1302 bed rest headboard 1 161 39 6 headboard otherfurniture Furniture n03502200 headboard.n.01 11 bed +1303 furniture 2 furniture 1 524 39 6 furniture otherfurniture Furniture n03405725 furniture.n.01 36 furniture +1304 wall ac vent vent 1 25 38 7 air vent otherstructure Objects n04526241 vent.n.01 40 misc +1305 dumbbells dumbbell 1 40 7 otherprop Objects n03255030 dumbbell.n.01 33 gym_equipment +1306 weight weight 1 40 7 otherprop Objects n04571292 weight.n.02 33 gym_equipment +1307 fixture fixture 1 40 7 otherprop Objects n03354613 fixture.n.01 39 objects +1308 llight light 1 144 35 7 lamp lamp Objects lamp lamp 03636649 n03636649 lamp.n.02 28 lighting +1309 show space show space 1 40 7 otherprop Objects 40 misc +1310 recessed cubby recessed cubby 1 40 7 otherprop Objects 40 misc +1311 oven hood range hood 1 380 38 7 range hood otherstructure Objects range_hood n04053677 range_hood.n.01 39 objects +1312 counter doors counter door 1 28 8 12 door door Wall door 40 misc +1313 roof grill roof grill 1 700 38 7 grill otherstructure Objects 40 misc +1314 shower dial shower dial 1 40 7 otherprop Objects 40 misc +1315 riser riser 1 40 7 otherprop Objects 40 misc +1316 plant ornament plant ornament 1 40 7 otherprop Objects 40 misc +1317 floor stand floor stand 1 50 39 6 stand otherfurniture Furniture 40 misc +1318 fruit fruit 1 286 40 7 fruit otherprop Objects n13134947 fruit.n.01 39 objects +1319 frige refrigerator 1 17 24 6 refridgerator refridgerator Furniture n04070727 refrigerator.n.01 37 appliances +1320 stairwell stairwell 1 40 7 otherprop Objects n04298661 stairwell.n.01 16 stairs +1321 trash bag trash bag 1 55 37 7 bag bag Objects 40 misc +1322 cups cup 1 35 40 7 cup otherprop Objects cup cup or mug 03797390 n03797390 mug.n.04 39 objects +1323 photo mount photo mount 1 40 7 otherprop Objects 40 misc +1324 drawers for clothes drawers for clothes 1 141 21 7 clothes clothes Objects 40 misc +1325 strange ceiling ceiling 1 4 22 3 ceiling ceiling Ceiling n02990373 ceiling.n.01 17 ceiling +1326 bilow billow 1 40 7 otherprop Objects 40 misc +1327 ceilng ceiling 1 4 22 3 ceiling ceiling Ceiling n02990373 ceiling.n.01 17 ceiling +1328 bathroom towel bathroom towel 1 135 27 7 towel towel Objects n04459362 towel.n.01 20 towel +1329 john john 1 40 7 otherprop Objects toilet toilet n04446276 toilet.n.01 18 toilet +1330 refrigerator cabinet refrigerator cabinet 1 3 3 6 cabinet cabinet Furniture cabinet 02933112 n02933112 cabinet.n.01 7 cabinet +1331 fish picture fish picture 1 64 11 8 picture picture Picture 40 misc +1332 painting frame painting frame 1 40 7 otherprop Objects 40 misc +1333 display table display table 1 19 7 10 table table Table table table table 04379243 n04379243 table.n.02 5 table +1334 round seat round seat 1 40 7 otherprop Objects 40 misc +1335 lower floor floor 1 11 2 5 floor floor Floor n03365592 floor.n.01 2 floor +1336 lid lid 1 533 40 7 lid otherprop Objects 40 misc +1337 bonsai tree bonsai tree 1 40 7 otherprop Objects 40 misc +1338 window3 window 1 59 9 13 window window Window n04587648 window.n.01 9 window +1339 window2 window 1 59 9 13 window window Window n04587648 window.n.01 9 window +1340 window1 window 1 59 9 13 window window Window n04587648 window.n.01 9 window +1341 window5 window 1 59 9 13 window window Window n04587648 window.n.01 9 window +1342 shower floor surround shower floor 1 11 2 5 floor floor Floor n04208936 shower.n.01 23 shower +1343 folders folder 1 69 40 7 folder otherprop Objects n03376279 folder.n.02 39 objects +1344 decorative plant decorative plant 1 82 40 7 plant otherprop Objects plant 40 misc +1345 cooker unit cooker unit 1 40 7 otherprop Objects 40 misc +1346 towel holder towel holder 1 40 7 otherprop Objects 40 misc +1347 laundrybag laundry bag 1 55 37 7 bag bag Objects 40 misc +1348 bathroom utencil bathroom utensil 1 267 40 7 utensil otherprop Objects n04516672 utensil.n.01 39 objects +1349 wine bottles wine bottle 1 333 40 7 wine bottle otherprop Objects bottle wine bottle 04591713 n04591713 wine_bottle.n.01 39 objects +1350 high shelf high shelf 1 40 7 otherprop Objects 40 misc +1351 loveseat couch 1 83 6 9 sofa sofa Sofa sofa sofa sofa 04256520 n04256520 sofa.n.01 10 sofa +1352 couch pillow pillow 1 119 18 7 pillow pillow Objects pillow 03938244 n03938244 pillow.n.01 8 cushion +1353 ceiling molding ceiling molding 1 40 7 otherprop Objects 40 misc +1354 firewook firewood 1 40 7 otherprop Objects n15100644 firewood.n.01 40 misc +1355 door /otherroom door /otherroom 1 28 8 12 door door Wall door n03221720 door.n.01 4 door +1356 toaster? toaster 1 251 40 7 toaster otherprop Objects n04442312 toaster.n.02 37 appliances +1357 foodstand food stand 1 50 39 6 stand otherfurniture Furniture 40 misc +1358 weight bench weight bench 1 457 39 6 excercise equipment otherfurniture Furniture n04285146 sports_equipment.n.01 33 gym_equipment +1359 floor / room above floor /otherroom 1 11 2 5 floor floor Floor n03365592 floor.n.01 2 floor +1360 mini fridge mini fridge 1 17 24 6 refridgerator refridgerator Furniture n03273913 electric_refrigerator.n.01 37 appliances +1361 bars bar 1 51 38 7 bar otherstructure Objects n02788689 bar.n.03 39 objects +1362 lion lion 1 594 40 7 cat otherprop Objects n02129165 lion.n.01 39 objects +1363 cuddly toy cuddly toy 1 389 40 7 toy otherprop Objects 40 misc +1364 toilet stall toilet stall 1 40 7 otherprop Objects 40 misc +1365 copier machine copier machine 1 220 40 7 machine otherprop Objects 40 misc +1366 raised platform platform 1 38 7 otherstructure Objects 40 misc +1367 bell bell 1 40 7 otherprop Objects 40 misc +1368 fireextinctioms fire extinguisher 1 10 40 7 fire extinguisher otherprop Objects n03345837 fire_extinguisher.n.01 39 objects +1369 table vase table vase 1 78 40 7 vase otherprop Objects vase 40 misc +1370 bedroom ceiling bedroom ceiling 1 4 22 3 ceiling ceiling Ceiling 40 misc +1371 ceiling \other room ceiling /otherroom 1 4 22 3 ceiling ceiling Ceiling n02990373 ceiling.n.01 17 ceiling +1372 chair parts chair part 1 40 7 otherprop Objects 40 misc +1373 door / window door/window 1 40 7 otherprop Objects 40 misc +1374 closet mirror wall closet mirror wall 1 21 1 12 wall wall Wall 40 misc +1375 beam across top of arch beam across top of arch 1 40 7 otherprop Objects 40 misc +1376 fireplace utensils fireplace utensil 1 267 40 7 utensil otherprop Objects 40 misc +1377 liight light 1 144 35 7 lamp lamp Objects lamp lamp 03636649 n03636649 lamp.n.02 28 lighting +1378 self side self side 1 40 7 otherprop Objects 40 misc +1379 kitchen faucet faucet 1 9 40 7 faucet otherprop Objects faucet 03325088 n03325088 faucet.n.01 39 objects +1380 candle holder candle holder 1 148 40 7 candlestick otherprop Objects n02948557 candlestick.n.01 39 objects +1381 mirror/other room mirror /otherroom 1 122 19 7 mirror mirror Objects n03773035 mirror.n.01 21 mirror +1382 hall top hall top 1 40 7 otherprop Objects 40 misc +1383 decoration / other room decoration /otherroom 1 40 7 otherprop Objects n03169390 decoration.n.01 39 objects +1384 shower hose shower hose 1 669 40 7 shower hose otherprop Objects 40 misc +1385 bottle of detergent bottle of detergent 1 40 7 otherprop Objects 40 misc +1386 box opening box opening 1 40 7 otherprop Objects 40 misc +1387 hunting trohpy hunting trophy 1 547 40 7 trophy otherprop Objects 40 misc +1388 rack with pool cues rack 1 50 39 6 stand otherfurniture Furniture n04038440 rack.n.05 31 shelving +1389 stack stack 1 40 7 otherprop Objects 40 misc +1390 bed stand bed stand 1 50 39 6 stand otherfurniture Furniture 40 misc +1391 planter curb planter curb 1 40 7 otherprop Objects 40 misc +1392 garage door motor garage door motor 1 40 7 otherprop Objects 40 misc +1393 entry entry 1 40 7 otherprop Objects n03290771 entrance.n.01 40 misc +1394 a-frame sign a-frame sign 1 208 40 7 sign otherprop Objects 40 misc +1395 cuboid cuboid 1 40 7 otherprop Objects 40 misc +1396 shelf with jars shelf 1 42 15 6 shelves shelves Furniture n04190052 shelf.n.01 31 shelving +1397 wall top wall top 1 40 7 otherprop Objects 40 misc +1398 window4 window 1 59 9 13 window window Window n04587648 window.n.01 9 window +1399 bath faucet bath faucet 1 9 40 7 faucet otherprop Objects faucet 03325088 n03325088 faucet.n.01 39 objects +1400 statue / art statue/art 1 40 7 otherprop Objects 40 misc +1401 cabinet /w clutter cabinet /w clutter 1 3 3 6 cabinet cabinet Furniture cabinet 02933112 n02933112 cabinet.n.01 7 cabinet +1402 workout bike workout bike 1 40 7 otherprop Objects 40 misc +1403 closet area for hanging clothes closet area for hanging clothes 1 141 21 7 clothes clothes Objects 40 misc +1404 vase \other room vase /otherroom 1 78 40 7 vase otherprop Objects vase jar 03593526 n04522168 vase.n.01 39 objects +1405 art / muscle shell art/muscle shell 1 40 7 otherprop Objects 40 misc +1406 center island island 1 456 38 7 kitchen island otherstructure Objects n03620600 kitchen_island.n.01 26 counter +1407 dedore decor 1 40 7 otherprop Objects n03579355 interior_decoration.n.01 39 objects +1408 dome roof dome roof 1 40 7 otherprop Objects 40 misc +1409 chair /w clutter chair /w clutter 1 5 5 4 chair chair Chair chair chair chair 03001627 n03001627 chair.n.01 3 chair +1410 basket of objects (popcorn?) basket 1 39 40 7 basket otherprop Objects basket 02801938 n02801938 basket.n.01 39 objects +1411 outer side wall wall 1 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +1412 top floor floor 1 11 2 5 floor floor Floor n03365592 floor.n.01 2 floor +1413 pelt pelt 1 40 7 otherprop Objects n01895735 hide.n.02 39 objects +1414 washer-dryer washer-dryer 1 40 7 otherprop Objects 40 misc +1415 yellow egg shaped vase yellow egg shaped vase 1 78 40 7 vase otherprop Objects vase 40 misc +1416 entry table table 1 19 7 10 table table Table table table table 04379243 n04379243 table.n.02 5 table +1417 domino decoration domino decoration 1 40 7 otherprop Objects 40 misc +1418 pc tower pc tower 1 40 7 otherprop Objects tower 04460130 n04460130 tower.n.01 40 misc +1419 unknown/ other room unknown /otherroom 1 20 40 7 unknown otherprop Objects n08632096 unknown.n.01 41 unlabeled +1420 bathroom glass bathroom glass 1 612 38 7 glass otherstructure Objects n03438257 glass.n.02 39 objects +1421 chanellier chandelier 1 342 38 7 chandelier otherstructure Objects n03005285 chandelier.n.01 28 lighting +1422 makeup area makeup area 1 40 7 otherprop Objects 40 misc +1423 fireplace mirror fireplace mirror 1 122 19 7 mirror mirror Objects 40 misc +1424 jug jug 1 687 40 7 jug otherprop Objects bottle bottle 02876657 n03603722 jug.n.01 39 objects +1425 bathroom window bathroom window 1 59 9 13 window window Window n04587648 window.n.01 9 window +1426 falling light falling light 1 62 38 7 light otherstructure Objects 40 misc +1427 snow globe snow globe 1 347 40 7 globe otherprop Objects 40 misc +1428 unknown kitchen appliance unknown kitchen appliance 1 40 7 otherprop Objects 40 misc +1429 boxes /w books boxes /w books 1 85 23 2 books books Books 40 misc +1430 alarm alarm 1 525 40 7 alarm otherprop Objects clock 03046257 n02694662 alarm_clock.n.01 39 objects +1431 electirc outlet electric outlet 1 98 40 7 electrical outlet otherprop Objects n04548771 wall_socket.n.01 39 objects +1432 sauna floor floor 1 11 2 5 floor floor Floor n03365592 floor.n.01 2 floor +1433 picture /otherroom picture /otherroom 1 64 11 8 picture picture Picture n03931044 picture.n.01 6 picture +1434 garage door railing garage door railing 1 497 38 7 railing otherstructure Objects 40 misc +1435 other ceiling ceiling 1 4 22 3 ceiling ceiling Ceiling n02990373 ceiling.n.01 17 ceiling +1436 lightbox frame light box frame 1 40 7 otherprop Objects 40 misc +1437 air hole vent 1 25 38 7 air vent otherstructure Objects n04526241 vent.n.01 40 misc +1438 mantel mantel 1 58 38 7 mantel otherstructure Objects mantel n03719343 mantel.n.01 27 fireplace +1439 cooker cooker 4 267 40 7 utensil otherprop Objects n03101156 cooker.n.01 39 objects +1440 partial partial 1 40 7 otherprop Objects 40 misc +1441 amplifier amplifier 1 40 7 otherprop Objects n02705944 amplifier.n.01 39 objects +1442 emergency exit emergency exit 1 40 7 otherprop Objects n03345658 fire_escape.n.01 16 stairs +1443 barbecue barbecue 1 40 7 otherprop Objects n02790669 barbecue.n.03 40 misc +1444 unkown / other room unknown /otherroom 1 20 40 7 unknown otherprop Objects n08632096 unknown.n.01 41 unlabeled +1445 bath sink bath sink 1 24 34 7 sink sink Objects sink 40 misc +1446 spa bench spa bench 1 204 39 6 bench otherfurniture Furniture bench bench 02828884 n02828884 bench.n.01 34 seating +1447 handsoap hand soap 1 133 40 7 soap otherprop Objects n04253437 soap.n.01 39 objects +1448 folding stand folding stand 1 50 39 6 stand otherfurniture Furniture 40 misc +1449 table plant table plant 1 82 40 7 plant otherprop Objects plant 40 misc +1450 hose outlet hose outlet 1 40 7 otherprop Objects 40 misc +1451 theater stage theater stage 1 40 7 otherprop Objects n04418818 theater_stage.n.01 39 objects +1452 bar cabinet bar cabinet 1 3 3 6 cabinet cabinet Furniture cabinet 02933112 n02933112 cabinet.n.01 7 cabinet +1453 washer and dryer washing machine and dryer 1 40 7 otherprop Objects 37 appliances +1454 door trim molding molding 1 38 7 otherstructure Objects n02800354 baseboard.n.01 1 wall +1455 paintings picture 1 64 11 8 picture picture Picture n03931044 picture.n.01 6 picture +1456 unknown wall part unknown wall 1 21 1 12 wall wall Wall 40 misc +1457 wooden balcony balcony 1 40 7 otherprop Objects 40 misc +1458 shelves with shoes shelving 1 42 15 6 shelves shelves Furniture n04190052 shelf.n.01 31 shelving +1459 stair landing landing 1 40 7 otherprop Objects n03638511 landing.n.01 2 floor +1460 rack of excercise weights rack 1 50 39 6 stand otherfurniture Furniture n04038440 rack.n.05 31 shelving +1461 unknown - hot tub? unknown - hot tub? 1 40 7 otherprop Objects 40 misc +1462 wall window window 1 59 9 13 window window Window n04587648 window.n.01 9 window +1463 ceiling smoke detector smoke detector 1 40 7 otherprop Objects 40 misc +1464 couch type chair sofa chair 1 5 5 4 chair chair Chair chair chair chair 03001627 n03001627 chair.n.01 3 chair +1465 scales scale 1 639 40 7 scale otherprop Objects n04141975 scale.n.07 39 objects +1466 chair rail chair rail 1 40 7 otherprop Objects 40 misc +1467 kitchen backsplash backsplash 1 40 7 otherprop Objects 40 misc +1468 shelves / other room shelves /otherroom 1 42 15 6 shelves shelves Furniture 40 misc +1469 tools tool 1 40 7 otherprop Objects n04451818 tool.n.01 39 objects +1470 hoses for chemical tank hoses for chemical tank 1 40 7 otherprop Objects 40 misc +1471 wall behind / remove wall /otherroom 1 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +1472 sauna heat rocks sauna heat rocks 1 40 7 otherprop Objects 40 misc +1473 camera camera 1 40 40 7 camera otherprop Objects camera 02942699 n02942699 camera.n.01 39 objects +1474 curtain lighter curtain lighter 1 40 7 otherprop Objects 40 misc +1475 table decor decoration 1 40 7 otherprop Objects n03169390 decoration.n.01 39 objects +1476 brochure brochure 1 69 40 7 folder otherprop Objects n06413889 booklet.n.01 39 objects +1477 flowerwage flower vase 1 78 40 7 vase otherprop Objects vase jar 03593526 n04522168 vase.n.01 39 objects +1478 drill drill 1 40 7 otherprop Objects 40 misc +1479 book stand bookshelf 1 88 10 6 bookshelf bookshelf Furniture bookshelf bookshelf 02871439 n02871439 bookshelf.n.01 31 shelving +1480 china cabinet cabinet 1 3 3 6 cabinet cabinet Furniture cabinet 02933112 n02933112 cabinet.n.01 7 cabinet +1481 remove / behind remove 1 0 0 0 void void void 0 void +1482 rotunda wall rotunda wall 1 21 1 12 wall wall Wall 40 misc +1483 stone support structure stone support structure 1 40 7 otherprop Objects 40 misc +1484 dresser decor decoration 1 40 7 otherprop Objects n03169390 decoration.n.01 39 objects +1485 roof / other room ceiling /otherroom 1 4 22 3 ceiling ceiling Ceiling n02990373 ceiling.n.01 17 ceiling +1486 elliptical elliptical 1 457 39 6 excercise equipment otherfurniture Furniture n04285146 sports_equipment.n.01 33 gym_equipment +1487 furnace furnace 1 551 39 6 furnace otherfurniture Furniture n03404449 furnace.n.01 40 misc +1488 foosball table foosball table 1 510 39 6 foosball table otherfurniture Furniture table table table 04379243 n04379243 table.n.02 5 table +1489 maracas maraca 1 40 7 otherprop Objects n03720891 maraca.n.01 39 objects +1490 box of fruit box of fruit 1 286 40 7 fruit otherprop Objects 40 misc +1491 window screen window screen 1 40 7 otherprop Objects n04589890 window_screen.n.01 9 window +1492 door tag door tag 1 218 40 7 tag otherprop Objects 40 misc +1493 fire pit fire pit 1 40 7 otherprop Objects n09280113 fire_pit.n.01 39 objects +1494 box of tissues box of tissues 1 40 7 otherprop Objects 40 misc +1495 portal portal 1 46 40 7 computer otherprop Objects n06359657 portal_site.n.01 39 objects +1496 yoga mat yoga mat 1 205 40 7 yoga mat otherprop Objects 40 misc +1497 toy duck toy duck 1 887 40 7 duck otherprop Objects 40 misc +1498 motorcycle motorcycle 1 40 7 otherprop Objects motorcycle 03790512 n03790512 motorcycle.n.01 39 objects +1499 computer tower computer tower 1 46 40 7 computer otherprop Objects n03082979 computer.n.01 39 objects +1500 frame for door frame for door 1 28 8 12 door door Wall door 40 misc +1501 theater screen theater screen 1 40 7 otherprop Objects 40 misc +1502 swimming pool swimming pool 1 40 7 otherprop Objects n04371225 swimming_pool.n.01 40 misc +1503 shelf with clutter shelf 1 42 15 6 shelves shelves Furniture n04190052 shelf.n.01 31 shelving +1504 spice rack spice rack 1 241 38 7 spice rack otherstructure Objects n04275175 spice_rack.n.01 31 shelving +1505 bed small bed small 1 40 7 otherprop Objects 40 misc +1506 object/other room object /otherroom 1 40 7 otherprop Objects n00002684 object.n.01 39 objects +1507 dinnerware dinnerware 1 40 7 otherprop Objects n03202622 dinnerware.n.01 39 objects +1508 storage shelves shelving 1 42 15 6 shelves shelves Furniture n04190052 shelf.n.01 31 shelving +1509 tanning bed tanning bed 1 157 4 1 bed bed Bed bed bed bed 02818832 n02818832 bed.n.01 11 bed +1510 grid door door 1 28 8 12 door door Wall door n03221720 door.n.01 4 door +1511 background chair chair 1 5 5 4 chair chair Chair chair chair chair 03001627 n03001627 chair.n.01 3 chair +1512 boarder boarder 1 40 7 otherprop Objects 40 misc +1513 closte closet 1 40 7 otherprop Objects n03148324 cupboard.n.01 7 cabinet +1514 doorframe / upstairs room doorframe /otherroom 1 615 38 7 door frame otherstructure Objects n03222722 doorframe.n.01 4 door +1515 rolling cart rolling cart 1 305 40 7 cart otherprop Objects 40 misc +1516 box of tissue box of tissue 1 648 40 7 tissue otherprop Objects 40 misc +1517 door nob door knob 1 27 40 7 door knob otherprop Objects 40 misc +1518 floor \other room floor /otherroom 1 11 2 5 floor floor Floor n03365592 floor.n.01 2 floor +1519 table w/ clutter table 1 19 7 10 table table Table table table table 04379243 n04379243 table.n.02 5 table +1520 table vase soil table vase soil 1 40 7 otherprop Objects 40 misc +1521 light box light box 1 26 29 7 box box Objects 40 misc +1522 wines wine 1 766 40 7 wine otherprop Objects 40 misc +1523 sliding glass door sliding glass door 1 28 8 12 door door Wall door 40 misc +1524 toilet handle toilet handle 1 758 40 7 handle otherprop Objects 40 misc +1525 bath deco bath deco 1 40 7 otherprop Objects 40 misc +1526 shower hose/head shower hose/head 1 40 7 otherprop Objects 40 misc +1527 shower case shower case 1 851 40 7 case otherprop Objects 40 misc +1528 mortar mortar 1 40 7 otherprop Objects 40 misc +1529 watering can watering can 1 329 40 7 can otherprop Objects can 02946921 n02946921 can.n.01 39 objects +1530 scarf scarf 1 240 40 7 scarf otherprop Objects n04143897 scarf.n.01 38 clothes +1531 softer softer 1 40 7 otherprop Objects 40 misc +1532 soap dish cubby soap dish cubby 1 40 7 otherprop Objects 40 misc +1533 paintintg picture 1 64 11 8 picture picture Picture n03931044 picture.n.01 6 picture +1534 cleaning clutter cleaning clutter 1 40 7 otherprop Objects 40 misc +1535 fireplace tool set fireplace tool set 1 40 7 otherprop Objects 40 misc +1536 table stand table stand 1 50 39 6 stand otherfurniture Furniture 40 misc +1537 shower valve shower valve 1 40 7 otherprop Objects 40 misc +1538 long sofa couch 1 83 6 9 sofa sofa Sofa sofa sofa sofa 04256520 n04256520 sofa.n.01 10 sofa +1539 wall soap shelf wall soap shelf 1 40 7 otherprop Objects 40 misc +1540 ceiling inset for fan ceiling inset for fan 1 74 40 7 fan otherprop Objects 40 misc +1541 beanbag chair beanbag chair 1 5 5 4 chair chair Chair chair chair chair 03001627 n03001627 chair.n.01 3 chair +1542 door bottom rail door bottom rail 1 40 7 otherprop Objects 40 misc +1543 sowing machine sowing machine 1 220 40 7 machine otherprop Objects 40 misc +1544 couch chair sofa chair 1 5 5 4 chair chair Chair chair chair chair 03001627 n03001627 chair.n.01 3 chair +1545 tabletop trinket tabletop trinket 1 844 40 7 trinket otherprop Objects 40 misc +1546 ceilling ceiling 3 4 22 3 ceiling ceiling Ceiling n02990373 ceiling.n.01 17 ceiling +1547 flowerstand flower stand 1 50 39 6 stand otherfurniture Furniture 40 misc +1548 wall stand wall stand 1 295 38 7 wall stand otherstructure Objects 40 misc +1549 shower cabin shower cabin 1 40 7 otherprop Objects 40 misc +1550 shower-bath cabinet shower-bath cabinet 1 3 3 6 cabinet cabinet Furniture cabinet 02933112 n02933112 cabinet.n.01 7 cabinet +1551 teddy bear stuffed animal 1 177 40 7 stuffed animal otherprop Objects n04399382 teddy.n.01 39 objects +1552 foor lamp floor lamp 1 144 35 7 lamp lamp Objects lamp lamp 03636649 n03367059 floor_lamp.n.01 28 lighting +1553 water fountain water fountain 1 339 38 7 water fountain otherstructure Objects n03241335 drinking_fountain.n.01 40 misc +1554 elephant sculpture elephant sculpture 1 294 40 7 sculpture otherprop Objects 40 misc +1555 grate grate 1 40 7 otherprop Objects 40 misc +1556 chicken chicken 1 40 7 otherprop Objects n01791625 chicken.n.02 39 objects +1557 bathroom fan bathroom fan 1 74 40 7 fan otherprop Objects n03320046 fan.n.01 39 objects +1558 roomba roomba 1 40 7 otherprop Objects 40 misc +1559 planter pot 1 16 40 7 pot otherprop Objects n03991062 pot.n.04 39 objects +1560 controls control 1 40 7 otherprop Objects n03096960 control.n.09 39 objects +1561 nightstand / other room nightstand /otherroom 1 158 32 6 night stand night stand Furniture night_stand night_stand n03015254 chest_of_drawers.n.01 13 chest_of_drawers +1562 worktop worktop 1 40 7 otherprop Objects 40 misc +1563 steps wall steps wall 1 21 1 12 wall wall Wall 40 misc +1564 door fireplace wall door 1 28 8 12 door door Wall door n03221720 door.n.01 4 door +1565 floor / upstairs room floor /otherroom 1 11 2 5 floor floor Floor n03365592 floor.n.01 2 floor +1566 oven and stove oven and stove 1 242 38 7 stove otherstructure Objects stove 04330267 40 misc +1567 chest drawer chest drawer 1 174 39 6 drawer otherfurniture Furniture 40 misc +1568 door outside door /outside 1 28 8 12 door door Wall door n03221720 door.n.01 4 door +1569 fireplace \other room fireplace /otherroom 1 372 38 7 fireplace otherstructure Objects n03346455 fireplace.n.01 27 fireplace +1570 hammock hammock 1 157 4 1 bed bed Bed bed bed bed 02818832 n03482252 hammock.n.02 11 bed +1571 dartboard dartboard 1 408 38 7 board otherstructure Objects n03162940 dartboard.n.01 39 objects +1572 fireplace brush fireplace brush 1 40 7 otherprop Objects 40 misc +1573 drum drum 1 145 40 7 drum otherprop Objects n03249569 drum.n.01 39 objects +1574 rotunda ceiling ceiling 1 4 22 3 ceiling ceiling Ceiling n02990373 ceiling.n.01 17 ceiling +1575 ceiling light fixture connection ceiling light fixture connection 1 40 7 otherprop Objects 40 misc +1576 stair fencing banister 1 453 38 7 banister otherstructure Objects n02788148 bannister.n.02 30 railing +1577 piano stool piano stool 1 150 40 7 stool otherprop Objects stool n03801880 music_stool.n.01 19 stool +1578 swing door swing door 1 28 8 12 door door Wall door n04371979 swing_door.n.01 4 door +1579 bathtub utencils bathtub utensil 1 267 40 7 utensil otherprop Objects 40 misc +1580 access area access area 1 40 7 otherprop Objects 40 misc +1581 exercise mat roll exercise mat roll 1 40 7 otherprop Objects 40 misc +1582 ceiling duct ceiling duct 1 4 22 3 ceiling ceiling Ceiling n02990373 ceiling.n.01 17 ceiling +1583 motorbike motorbike 1 40 7 otherprop Objects motorcycle 03790512 n03769722 minibike.n.01 39 objects +1584 floor elevator floor elevator 1 40 7 otherprop Objects 40 misc +1585 kitchen countertop items kitchen countertop items 1 40 7 otherprop Objects 40 misc +1586 entertainment set entertainment set 1 40 7 otherprop Objects 40 misc +1587 shower curtain bar shower curtain bar 1 51 38 7 bar otherstructure Objects 40 misc +1588 bedroom entry walls wall 1 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +1589 oven vent oven vent 1 40 7 otherprop Objects 40 misc +1590 shower floor /otherroom shower floor /otherroom 1 11 2 5 floor floor Floor n04208936 shower.n.01 23 shower +1591 washing bowl sink 1 24 34 7 sink sink Objects sink n04223580 sink.n.01 15 sink +1592 desert plate desert plate 1 233 40 7 plate otherprop Objects 40 misc +1593 fuse panel fuse panel 1 40 7 otherprop Objects 40 misc +1594 barbers chair barbers chair 1 5 5 4 chair chair Chair chair chair chair 03001627 n03001627 chair.n.01 3 chair +1595 diploma diploma 1 40 7 otherprop Objects 40 misc +1596 wall behind stove wall 1 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +1597 window6 window 1 59 9 13 window window Window n04587648 window.n.01 9 window +1598 ceiling /otherroom ceiling /otherroom 1 4 22 3 ceiling ceiling Ceiling n02990373 ceiling.n.01 17 ceiling +1599 ventilation hood ventilation hood 1 40 7 otherprop Objects 40 misc +1600 dirt ground dirt ground 1 40 7 otherprop Objects 40 misc +1601 awning awning 1 40 7 otherprop Objects n02763901 awning.n.01 39 objects +1602 stiarcase step step 1 38 7 otherstructure Objects n04314914 step.n.04 16 stairs +1603 rack of weights rack of weights 1 40 7 otherprop Objects 40 misc +1604 shower tub shower tub 1 675 40 7 shower tube otherprop Objects bathtub bathtub tub 02808440 n02808440 bathtub.n.01 25 bathtub +1605 washing powder washing powder 1 40 7 otherprop Objects 40 misc +1606 toilet cleaner toilet cleaner 1 548 40 7 cleaner otherprop Objects 40 misc +1607 soap displensor shelf in shower soap dispenser shelf in shower 1 40 7 otherprop Objects 40 misc +1608 window or door window/door 1 40 7 otherprop Objects 40 misc +1609 storage boxes storage box 1 26 29 7 box box Objects 40 misc +1610 closet storage area closet storage area 1 40 7 otherprop Objects 40 misc +1611 stationary bike exercise bike 1 457 39 6 excercise equipment otherfurniture Furniture n03302671 exercise_bike.n.01 33 gym_equipment +1612 bedding bedding 1 40 7 otherprop Objects n02820210 bedclothes.n.01 39 objects +1613 stair handrail banister 1 453 38 7 banister otherstructure Objects n02788148 bannister.n.02 30 railing +1614 room divider partition 1 21 1 12 wall wall Wall n03894379 partition.n.01 40 misc +1615 window frames window frame 1 59 9 13 window window Window n04589593 window_frame.n.01 9 window +1616 wall electrics wall electronics 1 40 7 otherprop Objects 40 misc +1617 other step step 1 38 7 otherstructure Objects n04314914 step.n.04 16 stairs +1618 tabletop objects object 1 40 7 otherprop Objects n00002684 object.n.01 39 objects +1619 foosball game table foosball game table 1 19 7 10 table table Table table table table 04379243 n04379243 table.n.02 5 table +1620 wall showcase wall showcase 1 40 7 otherprop Objects 40 misc +1621 kitchen utencil kitchen utensil 1 267 40 7 utensil otherprop Objects n03621049 kitchen_utensil.n.01 39 objects +1622 towels in a basket towels in a basket 1 39 40 7 basket otherprop Objects basket 02801938 n02801938 basket.n.01 39 objects +1623 lamp stand lamp stand 1 50 39 6 stand otherfurniture Furniture 40 misc +1624 toy giraffe toy giraffe 1 40 7 otherprop Objects 40 misc +1625 big door frame door frame 1 28 8 12 door door Wall door n03221720 door.n.01 4 door +1626 unknown / probably shrubbery unknown/probably shrubbery 1 40 7 otherprop Objects 40 misc +1627 wooden wall paneling paneling 1 21 1 12 wall wall Wall n03882611 paneling.n.01 1 wall +1628 bedstead bedstead 1 157 4 1 bed bed Bed n02822579 bedstead.n.01 11 bed +1629 fire dish fire dish 1 40 7 otherprop Objects 40 misc +1630 tray with tea cups tray 1 179 40 7 tray otherprop Objects n04476259 tray.n.01 39 objects +1631 television table tv stand 1 291 39 6 tv stand otherfurniture Furniture tv_stand n03290653 entertainment_center.n.01 36 furniture +1632 tissues tissue 1 648 40 7 tissue otherprop Objects 40 misc +1633 sitting area sitting area 1 40 7 otherprop Objects 40 misc +1634 wall vent vent 1 25 38 7 air vent otherstructure Objects n04526241 vent.n.01 40 misc +1635 utensils utensil 1 267 40 7 utensil otherprop Objects n04516672 utensil.n.01 39 objects +1636 pip pip 1 286 40 7 fruit otherprop Objects n11685091 pip.n.03 39 objects +1637 fireplace wall fireplace wall 1 21 1 12 wall wall Wall 40 misc +1638 stonework stonework 1 40 7 otherprop Objects n04326799 stonework.n.01 40 misc +1639 bottom of stairs bottom of stairs 1 215 38 7 stairs otherstructure Objects stairs 40 misc +1640 star star 1 40 7 otherprop Objects n09444783 star.n.03 39 objects +1641 art / deer statue statue 1 294 40 7 sculpture otherprop Objects n04306847 statue.n.01 39 objects +1642 unknown objects object 1 40 7 otherprop Objects n00002684 object.n.01 39 objects +1643 wall side wall 1 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +1644 shower glass /shifted shower glass /shifted 1 612 38 7 glass otherstructure Objects 40 misc +1645 toilet stall partition partition 1 21 1 12 wall wall Wall n03894379 partition.n.01 40 misc +1646 ceiling design 1 40 7 otherprop Objects 40 misc +1647 conference table conference table 1 19 7 10 table table Table table table table 04379243 n03090000 conference_table.n.01 5 table +1648 edge edge 1 40 7 otherprop Objects 40 misc +1649 electrical box electrical box 1 26 29 7 box box Objects n03034244 circuit_breaker.n.01 39 objects +1650 entrance entrance 1 40 7 otherprop Objects n03290771 entrance.n.01 40 misc +1651 gap gap 2 40 7 otherprop Objects n09249034 col.n.01 39 objects +1652 garage door opener motor garage door opener motor 1 40 7 otherprop Objects 40 misc +1653 lamp base lamp 1 144 35 7 lamp lamp Objects lamp lamp 03636649 n03636649 lamp.n.02 28 lighting +1654 racks rack 2 50 39 6 stand otherfurniture Furniture n04038440 rack.n.05 31 shelving +1655 stringer stringer 1 40 7 otherprop Objects 40 misc +1656 supporting structure supporting structure 1 40 7 otherprop Objects n04361095 supporting_structure.n.01 40 misc +1657 typewriter typewriter 1 376 40 7 typewriter otherprop Objects printer 04004475 n04505036 typewriter.n.01 39 objects +1658 walls wall 4 21 1 12 wall wall Wall n04546855 wall.n.01 1 wall +1659 washbasin top washbasin 1 24 34 7 sink sink Objects sink n04553920 washbasin.n.01 15 sink \ No newline at end of file diff --git a/MaskClustering/preprocess/matterport3d/constants.py b/MaskClustering/preprocess/matterport3d/constants.py new file mode 100755 index 0000000000000000000000000000000000000000..bdf3ed2ff16e457a5963cfb1c84430f8a4cd10b4 --- /dev/null +++ b/MaskClustering/preprocess/matterport3d/constants.py @@ -0,0 +1,13 @@ +import pandas as pd +import numpy as np +category_mapping = pd.read_csv("category_mapping.tsv", sep='\t', header=0) +RAW_TO_NYU = np.concatenate([[0], category_mapping[['nyuId']].to_numpy().flatten()]) + +MATTERPORT_VALID_IDS = [21, 28, 4, 11, 64, 59, 5, 119, 144, 3, 89, 19, 82, 122, 135, 24, 42, 83, 157, 158, 124, 94, 453, + 215, 150, 78, 172, 16, 36, 26, 356, 7, 204, 12, 372, 141, 136, 1, 25, 9, 508, 139, 74, 497, 294, + 169, 130, 359, 2, 17, 88, 772, 41, 49, 50, 174, 140, 301, 181, 609, 39, 342, 238, 56, 242, 278, + 123, 338, 307, 344, 13, 80, 22, 138, 233, 291, 149, 111, 161, 427, 137, 146, 54, 524, 208, 79, + 10, 582, 143, 66, 32, 312, 758, 650, 133, 47, 110, 236, 456, 113, 559, 612, 8, 35, 48, 850, 193, + 86, 298, 408, 560, 60, 457, 211, 148, 62, 639, 55, 37, 458, 300, 540, 647, 51, 179, 151, 383, 515, + 324, 502, 509, 267, 678, 177, 14, 859, 530, 630, 99, 145, 45, 380, 605, 389, 163, 638, 154, 548, + 46, 652, 15, 90, 400, 851, 589, 783, 844, 702, 331, 525] \ No newline at end of file diff --git a/MaskClustering/preprocess/matterport3d/process.py b/MaskClustering/preprocess/matterport3d/process.py new file mode 100644 index 0000000000000000000000000000000000000000..3cb3de35fc529f5bb5124d4ed261a134876655a6 --- /dev/null +++ b/MaskClustering/preprocess/matterport3d/process.py @@ -0,0 +1,75 @@ +import os +import numpy as np +from plyfile import PlyData +import tqdm +import json +from constants import RAW_TO_NYU, MATTERPORT_VALID_IDS + +raw_data_dir = '../../data/matterport3d/scans' +gt_dir = '../../data/matterport3d/gt' +split_file_path = '../../splits/matterport3d.txt' + +def load_instance_json(filepath): + with open(filepath, 'r') as f: + info = json.load(f) + instances = info["segGroups"] + instance_segments_id = [] + for instance in instances: + instance_segments_id.append(np.array(instance["segments"])) + return instance_segments_id + +def load_face_json(filepath): + with open(filepath, 'r') as f: + info = json.load(f) + return np.array(info["segIndices"]) + +def load_ply(filepath): + with open(filepath, "rb") as f: + plydata = PlyData.read(f) + vdata = plydata.elements[0].data # vertex + coords = np.array([vdata["x"], vdata["y"], vdata["z"]], dtype=np.float32).T + + fdata = plydata['face'].data + faces = np.stack(fdata['vertex_indices']) + + face_semantic_id = np.array(fdata["category_id"], dtype=np.int32) + vert_semantic_id = np.zeros(coords.shape[0], dtype=np.int32) + vert_semantic_id[faces.reshape(-1)] = face_semantic_id[None].repeat(3, axis=1).reshape(-1) + + return vert_semantic_id, faces + +def convert_gt(root_dir, seq_name, output_dir): + scene_dir = os.path.join(root_dir, seq_name, seq_name) + vert_semantic_id, faces = load_ply(os.path.join(scene_dir, 'house_segmentations', f'{seq_name}.ply')) + + face_segment_id = load_face_json(os.path.join(scene_dir, 'house_segmentations', f'{seq_name}.fsegs.json')) + vert_segment_id = np.zeros_like(vert_semantic_id) + vert_segment_id[faces.reshape(-1)] = face_segment_id[None].repeat(3, axis=1).reshape(-1) + + segment_ids = np.unique(vert_segment_id) + instance_segments_id = load_instance_json(os.path.join(scene_dir, 'house_segmentations', f'{seq_name}.semseg.json')) + segment_instance_id = np.full(segment_ids.max() + 1, -1) + for instance_id, segments_id in enumerate(instance_segments_id): + segment_instance_id[segments_id] = instance_id + + vert_instance_id = segment_instance_id[vert_segment_id] + + assert vert_instance_id.shape == vert_segment_id.shape + assert vert_instance_id.min() >= 0 and vert_instance_id.max() <= len(instance_segments_id) + + vert_semantic_id[vert_semantic_id < 0] = 0 + vert_semantic_id = RAW_TO_NYU[vert_semantic_id] + vert_semantic_id[np.isin(vert_semantic_id, MATTERPORT_VALID_IDS, invert=True)] = 0 + + print(np.unique(vert_semantic_id).shape[0], ' semantics, ', len(instance_segments_id), ' instances') + gt_id = vert_semantic_id * 1000 + vert_instance_id + 1 + + os.makedirs(output_dir, exist_ok=True) + np.savetxt(os.path.join(output_dir, f'{seq_name}.txt'), gt_id.astype(np.int32), fmt="%d") + +if __name__ == "__main__": + with open(split_file_path, 'r') as f: + seq_name_list = f.readlines() + seq_name_list = [seq_name.strip() for seq_name in seq_name_list] + for seq_name in tqdm.tqdm(seq_name_list): + convert_gt(raw_data_dir, seq_name, gt_dir) \ No newline at end of file diff --git a/MaskClustering/preprocess/scannet/SensorData.py b/MaskClustering/preprocess/scannet/SensorData.py new file mode 100644 index 0000000000000000000000000000000000000000..e8d1f05e52381da20985c239dd7e69f776278e02 --- /dev/null +++ b/MaskClustering/preprocess/scannet/SensorData.py @@ -0,0 +1,126 @@ + +import os, struct +import numpy as np +import zlib +import imageio +import cv2 +import png +from tqdm import tqdm + +COMPRESSION_TYPE_COLOR = {-1:'unknown', 0:'raw', 1:'png', 2:'jpeg'} +COMPRESSION_TYPE_DEPTH = {-1:'unknown', 0:'raw_ushort', 1:'zlib_ushort', 2:'occi_ushort'} + +class RGBDFrame(): + + def load(self, file_handle): + self.camera_to_world = np.asarray(struct.unpack('f'*16, file_handle.read(16*4)), dtype=np.float32).reshape(4, 4) + self.timestamp_color = struct.unpack('Q', file_handle.read(8))[0] + self.timestamp_depth = struct.unpack('Q', file_handle.read(8))[0] + self.color_size_bytes = struct.unpack('Q', file_handle.read(8))[0] + self.depth_size_bytes = struct.unpack('Q', file_handle.read(8))[0] + self.color_data = b''.join(struct.unpack('c'*self.color_size_bytes, file_handle.read(self.color_size_bytes))) + self.depth_data = b''.join(struct.unpack('c'*self.depth_size_bytes, file_handle.read(self.depth_size_bytes))) + + + def decompress_depth(self, compression_type): + if compression_type == 'zlib_ushort': + return self.decompress_depth_zlib() + else: + raise + + + def decompress_depth_zlib(self): + return zlib.decompress(self.depth_data) + + + def decompress_color(self, compression_type): + if compression_type == 'jpeg': + return self.decompress_color_jpeg() + else: + raise + + + def decompress_color_jpeg(self): + return imageio.imread(self.color_data) + + +class SensorData: + + def __init__(self, filename): + self.version = 4 + self.load(filename) + + + def load(self, filename): + with open(filename, 'rb') as f: + version = struct.unpack('I', f.read(4))[0] + assert self.version == version + strlen = struct.unpack('Q', f.read(8))[0] + self.sensor_name = b''.join(struct.unpack('c'*strlen, f.read(strlen))) + self.intrinsic_color = np.asarray(struct.unpack('f'*16, f.read(16*4)), dtype=np.float32).reshape(4, 4) + self.extrinsic_color = np.asarray(struct.unpack('f'*16, f.read(16*4)), dtype=np.float32).reshape(4, 4) + self.intrinsic_depth = np.asarray(struct.unpack('f'*16, f.read(16*4)), dtype=np.float32).reshape(4, 4) + self.extrinsic_depth = np.asarray(struct.unpack('f'*16, f.read(16*4)), dtype=np.float32).reshape(4, 4) + self.color_compression_type = COMPRESSION_TYPE_COLOR[struct.unpack('i', f.read(4))[0]] + self.depth_compression_type = COMPRESSION_TYPE_DEPTH[struct.unpack('i', f.read(4))[0]] + self.color_width = struct.unpack('I', f.read(4))[0] + self.color_height = struct.unpack('I', f.read(4))[0] + self.depth_width = struct.unpack('I', f.read(4))[0] + self.depth_height = struct.unpack('I', f.read(4))[0] + self.depth_shift = struct.unpack('f', f.read(4))[0] + num_frames = struct.unpack('Q', f.read(8))[0] + self.frames = [] + for i in tqdm(range(num_frames)): + frame = RGBDFrame() + frame.load(f) + self.frames.append(frame) + + + def export_depth_images(self, output_path, image_size=None, frame_skip=1): + if not os.path.exists(output_path): + os.makedirs(output_path) + print ('exporting', len(self.frames)//frame_skip, ' depth frames to', output_path) + for f in range(0, len(self.frames), frame_skip): + depth_data = self.frames[f].decompress_depth(self.depth_compression_type) + depth = np.fromstring(depth_data, dtype=np.uint16).reshape(self.depth_height, self.depth_width) + if image_size is not None: + depth = cv2.resize(depth, (image_size[1], image_size[0]), interpolation=cv2.INTER_NEAREST) + #imageio.imwrite(os.path.join(output_path, str(f) + '.png'), depth) + with open(os.path.join(output_path, str(f) + '.png'), 'wb') as f: # write 16-bit + writer = png.Writer(width=depth.shape[1], height=depth.shape[0], bitdepth=16) + depth = depth.reshape(-1, depth.shape[1]).tolist() + writer.write(f, depth) + + def export_color_images(self, output_path, image_size=None, frame_skip=1): + if not os.path.exists(output_path): + os.makedirs(output_path) + print ('exporting', len(self.frames)//frame_skip, 'color frames to', output_path) + for f in range(0, len(self.frames), frame_skip): + color = self.frames[f].decompress_color(self.color_compression_type) + if image_size is not None: + color = cv2.resize(color, (image_size[1], image_size[0]), interpolation=cv2.INTER_NEAREST) + imageio.imwrite(os.path.join(output_path, str(f) + '.jpg'), color) + + + def save_mat_to_file(self, matrix, filename): + with open(filename, 'w') as f: + for line in matrix: + np.savetxt(f, line[np.newaxis], fmt='%f') + + + def export_poses(self, output_path, frame_skip=1): + if not os.path.exists(output_path): + os.makedirs(output_path) + print ('exporting', len(self.frames)//frame_skip, 'camera poses to', output_path) + for f in range(0, len(self.frames), frame_skip): + self.save_mat_to_file(self.frames[f].camera_to_world, os.path.join(output_path, str(f) + '.txt')) + + + def export_intrinsics(self, output_path): + if not os.path.exists(output_path): + os.makedirs(output_path) + print ('exporting camera intrinsics to', output_path) + self.save_mat_to_file(self.intrinsic_color, os.path.join(output_path, 'intrinsic_color.txt')) + self.save_mat_to_file(self.extrinsic_color, os.path.join(output_path, 'extrinsic_color.txt')) + self.save_mat_to_file(self.intrinsic_depth, os.path.join(output_path, 'intrinsic_depth.txt')) + self.save_mat_to_file(self.extrinsic_depth, os.path.join(output_path, 'extrinsic_depth.txt')) diff --git a/MaskClustering/preprocess/scannet/download-scannet.py b/MaskClustering/preprocess/scannet/download-scannet.py new file mode 100644 index 0000000000000000000000000000000000000000..fd7be98b724e607335ba9b3e58d70a75fb3ace27 --- /dev/null +++ b/MaskClustering/preprocess/scannet/download-scannet.py @@ -0,0 +1,243 @@ +#!/usr/bin/env python +# Downloads ScanNet public data release +# Run with ./download-scannet.py (or python download-scannet.py on Windows) +# -*- coding: utf-8 -*- +import argparse +import os +import urllib.request +import tempfile +from tqdm import tqdm + +import ssl +ssl._create_default_https_context = ssl._create_unverified_context + +BASE_URL = 'http://kaldir.vc.in.tum.de/scannet/' +TOS_URL = BASE_URL + 'ScanNet_TOS.pdf' +FILETYPES = ['.aggregation.json', '.sens', '.txt', '_vh_clean.ply', '_vh_clean_2.0.010000.segs.json', '_vh_clean_2.ply', '_vh_clean.segs.json', '_vh_clean.aggregation.json', '_vh_clean_2.labels.ply', '_2d-instance.zip', '_2d-instance-filt.zip', '_2d-label.zip', '_2d-label-filt.zip'] +FILETYPES_TEST = ['.sens', '.txt', '_vh_clean.ply', '_vh_clean_2.ply'] +PREPROCESSED_FRAMES_FILE = ['scannet_frames_25k.zip', '5.6GB'] +TEST_FRAMES_FILE = ['scannet_frames_test.zip', '610MB'] +LABEL_MAP_FILES = ['scannetv2-labels.combined.tsv', 'scannet-labels.combined.tsv'] +DATA_EFFICIENT_FILES = ['limited-reconstruction-scenes.zip', 'limited-annotation-points.zip', 'limited-bboxes.zip', '1.7MB'] +GRIT_FILES = ['ScanNet-GRIT.zip'] +RELEASES = ['v2/scans', 'v1/scans'] +RELEASES_TASKS = ['v2/tasks', 'v1/tasks'] +RELEASES_NAMES = ['v2', 'v1'] +RELEASE = RELEASES[0] +RELEASE_TASKS = RELEASES_TASKS[0] +RELEASE_NAME = RELEASES_NAMES[0] +LABEL_MAP_FILE = LABEL_MAP_FILES[0] +RELEASE_SIZE = '1.2TB' +V1_IDX = 1 + + +def get_release_scans(release_file): + scan_lines = urllib.request.urlopen(release_file) + scans = [] + for scan_line in scan_lines: + scan_id = scan_line.decode('utf8').rstrip('\n') + scans.append(scan_id) + return scans + + +def download_release(release_scans, out_dir, file_types, use_v1_sens, skip_existing): + if len(release_scans) == 0: + return + print('Downloading ScanNet ' + RELEASE_NAME + ' release to ' + out_dir + '...') + for scan_id in release_scans: + scan_out_dir = os.path.join(out_dir, scan_id) + download_scan(scan_id, scan_out_dir, file_types, use_v1_sens, skip_existing) + print('Downloaded ScanNet ' + RELEASE_NAME + ' release.') + + +def download_file(url, out_file): + out_dir = os.path.dirname(out_file) + if not os.path.isdir(out_dir): + os.makedirs(out_dir) + if not os.path.isfile(out_file): + print('\t' + url + ' > ' + out_file) + fh, out_file_tmp = tempfile.mkstemp(dir=out_dir) + f = os.fdopen(fh, 'w') + f.close() + urllib.request.urlretrieve(url, out_file_tmp) + os.rename(out_file_tmp, out_file) + else: + print('WARNING: skipping download of existing file ' + out_file) + +def download_scan(scan_id, out_dir, file_types, use_v1_sens, skip_existing=False): + print('Downloading ScanNet ' + RELEASE_NAME + ' scan ' + scan_id + ' ...') + if not os.path.isdir(out_dir): + os.makedirs(out_dir) + for ft in file_types: + v1_sens = use_v1_sens and ft == '.sens' + url = BASE_URL + RELEASE + '/' + scan_id + '/' + scan_id + ft if not v1_sens else BASE_URL + RELEASES[V1_IDX] + '/' + scan_id + '/' + scan_id + ft + out_file = out_dir + '/' + scan_id + ft + if skip_existing and os.path.isfile(out_file): + continue + download_file(url, out_file) + print('Downloaded scan ' + scan_id) + + +def download_task_data(out_dir): + print('Downloading ScanNet v1 task data...') + files = [ + LABEL_MAP_FILES[V1_IDX], 'obj_classification/data.zip', + 'obj_classification/trained_models.zip', 'voxel_labeling/data.zip', + 'voxel_labeling/trained_models.zip' + ] + for file in files: + url = BASE_URL + RELEASES_TASKS[V1_IDX] + '/' + file + localpath = os.path.join(out_dir, file) + localdir = os.path.dirname(localpath) + if not os.path.isdir(localdir): + os.makedirs(localdir) + download_file(url, localpath) + print('Downloaded task data.') + +def download_tfrecords(in_dir, out_dir): + print('Downloading tf records (302 GB)...') + if not os.path.exists(out_dir): + os.makedirs(out_dir) + split_to_num_shards = {'train': 100, 'val': 25, 'test': 10} + + for folder_name in ['hires_tfrecords', 'lores_tfrecords']: + folder_dir = '%s/%s' % (in_dir, folder_name) + save_dir = '%s/%s' % (out_dir, folder_name) + if not os.path.exists(save_dir): + os.makedirs(save_dir) + for split, num_shards in split_to_num_shards.items(): + for i in range(num_shards): + file_name = '%s-%05d-of-%05d.tfrecords' % (split, i, num_shards) + url = '%s/%s' % (folder_dir, file_name) + localpath = '%s/%s/%s' % (out_dir, folder_name, file_name) + download_file(url, localpath) + +def download_label_map(out_dir): + print('Downloading ScanNet ' + RELEASE_NAME + ' label mapping file...') + files = [ LABEL_MAP_FILE ] + for file in files: + url = BASE_URL + RELEASE_TASKS + '/' + file + localpath = os.path.join(out_dir, file) + localdir = os.path.dirname(localpath) + if not os.path.isdir(localdir): + os.makedirs(localdir) + download_file(url, localpath) + print('Downloaded ScanNet ' + RELEASE_NAME + ' label mapping file.') + +def input(*args, **kwargs): + return 'y' + +def main(): + parser = argparse.ArgumentParser(description='Downloads ScanNet public data release.') + parser.add_argument('-o', '--out_dir', required=True, help='directory in which to download') + parser.add_argument('--task_data', action='store_true', help='download task data (v1)') + parser.add_argument('--label_map', action='store_true', help='download label map file') + parser.add_argument('--v1', action='store_true', help='download ScanNet v1 instead of v2') + parser.add_argument('--ids', help='specific scan ids to download', nargs='+') + parser.add_argument('--preprocessed_frames', action='store_true', help='download preprocessed subset of ScanNet frames (' + PREPROCESSED_FRAMES_FILE[1] + ')') + parser.add_argument('--test_frames_2d', action='store_true', help='download 2D test frames (' + TEST_FRAMES_FILE[1] + '; also included with whole dataset download)') + parser.add_argument('--data_efficient', action='store_true', help='download data efficient task files; also included with whole dataset download)') + parser.add_argument('--tf_semantic', action='store_true', help='download google tensorflow records for 3D segmentation / detection') + parser.add_argument('--grit', action='store_true', help='download ScanNet files for General Robust Image Task') + parser.add_argument('--type', help='specific file type to download (.aggregation.json, .sens, .txt, _vh_clean.ply, _vh_clean_2.0.010000.segs.json, _vh_clean_2.ply, _vh_clean.segs.json, _vh_clean.aggregation.json, _vh_clean_2.labels.ply, _2d-instance.zip, _2d-instance-filt.zip, _2d-label.zip, _2d-label-filt.zip)') + parser.add_argument('--skip_existing', action='store_true', help='skip download of existing files when downloading full release') + args = parser.parse_args() + + print('By pressing any key to continue you confirm that you have agreed to the ScanNet terms of use as described at:') + print(TOS_URL) + print('***') + print('Press any key to continue, or CTRL-C to exit.') + key = input('') + + if args.v1: + global RELEASE + global RELEASE_TASKS + global RELEASE_NAME + global LABEL_MAP_FILE + RELEASE = RELEASES[V1_IDX] + RELEASE_TASKS = RELEASES_TASKS[V1_IDX] + RELEASE_NAME = RELEASES_NAMES[V1_IDX] + LABEL_MAP_FILE = LABEL_MAP_FILES[V1_IDX] + assert((not args.tf_semantic) and (not args.grit)), "Task files specified invalid for v1" + + release_file = BASE_URL + RELEASE + '.txt' + release_scans = get_release_scans(release_file) + file_types = FILETYPES; + release_test_file = BASE_URL + RELEASE + '_test.txt' + release_test_scans = [] if args.v1 else get_release_scans(release_test_file) + file_types_test = FILETYPES_TEST; + out_dir_scans = os.path.join(args.out_dir, 'scans') + out_dir_test_scans = os.path.join(args.out_dir, 'scans_test') + out_dir_tasks = os.path.join(args.out_dir, 'tasks') + + if args.type: # download file type + file_type = args.type + if file_type not in FILETYPES: + print('ERROR: Invalid file type: ' + file_type) + return + file_types = [file_type] + if file_type in FILETYPES_TEST: + file_types_test = [file_type] + else: + file_types_test = [] + if args.task_data: # download task data + download_task_data(out_dir_tasks) + elif args.label_map: # download label map file + download_label_map(args.out_dir) + elif args.preprocessed_frames: # download preprocessed scannet_frames_25k.zip file + if args.v1: + print('ERROR: Preprocessed frames only available for ScanNet v2') + print('You are downloading the preprocessed subset of frames ' + PREPROCESSED_FRAMES_FILE[0] + ' which requires ' + PREPROCESSED_FRAMES_FILE[1] + ' of space.') + download_file(os.path.join(BASE_URL, RELEASE_TASKS, PREPROCESSED_FRAMES_FILE[0]), os.path.join(out_dir_tasks, PREPROCESSED_FRAMES_FILE[0])) + elif args.test_frames_2d: # download test scannet_frames_test.zip file + if args.v1: + print('ERROR: 2D test frames only available for ScanNet v2') + print('You are downloading the 2D test set ' + TEST_FRAMES_FILE[0] + ' which requires ' + TEST_FRAMES_FILE[1] + ' of space.') + download_file(os.path.join(BASE_URL, RELEASE_TASKS, TEST_FRAMES_FILE[0]), os.path.join(out_dir_tasks, TEST_FRAMES_FILE[0])) + elif args.data_efficient: # download data efficient task files + print('You are downloading the data efficient task files' + ' which requires ' + DATA_EFFICIENT_FILES[-1] + ' of space.') + for k in range(len(DATA_EFFICIENT_FILES)-1): + download_file(os.path.join(BASE_URL, RELEASE_TASKS, DATA_EFFICIENT_FILES[k]), os.path.join(out_dir_tasks, DATA_EFFICIENT_FILES[k])) + elif args.tf_semantic: # download google tf records + download_tfrecords(os.path.join(BASE_URL, RELEASE_TASKS, 'tf3d'), os.path.join(out_dir_tasks, 'tf3d')) + elif args.grit: # download GRIT file + download_file(os.path.join(BASE_URL, RELEASE_TASKS, GRIT_FILES[0]), os.path.join(out_dir_tasks, GRIT_FILES[0])) + elif args.ids: # download single scan + for scan_id in tqdm(args.ids): + is_test_scan = scan_id in release_test_scans + if scan_id not in release_scans and (not is_test_scan or args.v1): + print('ERROR: Invalid scan id: ' + scan_id) + else: + out_dir = os.path.join(out_dir_scans, scan_id) if not is_test_scan else os.path.join(out_dir_test_scans, scan_id) + scan_file_types = file_types if not is_test_scan else file_types_test + use_v1_sens = not is_test_scan + if not is_test_scan and not args.v1 and '.sens' in scan_file_types: + print('Note: ScanNet v2 uses the same .sens files as ScanNet v1: Press \'n\' to exclude downloading .sens files for each scan') + key = input('') + if key.strip().lower() == 'n': + scan_file_types.remove('.sens') + download_scan(scan_id, out_dir, scan_file_types, use_v1_sens, skip_existing=args.skip_existing) + else: # download entire release + if len(file_types) == len(FILETYPES): + print('WARNING: You are downloading the entire ScanNet ' + RELEASE_NAME + ' release which requires ' + RELEASE_SIZE + ' of space.') + else: + print('WARNING: You are downloading all ScanNet ' + RELEASE_NAME + ' scans of type ' + file_types[0]) + print('Note that existing scan directories will be skipped. Delete partially downloaded directories to re-download.') + print('***') + print('Press any key to continue, or CTRL-C to exit.') + key = input('') + if not args.v1 and '.sens' in file_types: + print('Note: ScanNet v2 uses the same .sens files as ScanNet v1: Press \'n\' to exclude downloading .sens files for each scan') + key = input('') + if key.strip().lower() == 'n': + file_types.remove('.sens') + download_release(release_scans, out_dir_scans, file_types, use_v1_sens=True, skip_existing=args.skip_existing) + if not args.v1: + download_label_map(args.out_dir) + download_release(release_test_scans, out_dir_test_scans, file_types_test, use_v1_sens=False, skip_existing=args.skip_existing) + download_file(os.path.join(BASE_URL, RELEASE_TASKS, TEST_FRAMES_FILE[0]), os.path.join(out_dir_tasks, TEST_FRAMES_FILE[0])) + for k in range(len(DATA_EFFICIENT_FILES)-1): + download_file(os.path.join(BASE_URL, RELEASE_TASKS, DATA_EFFICIENT_FILES[k]), os.path.join(out_dir_tasks, DATA_EFFICIENT_FILES[k])) + + +if __name__ == "__main__": main() diff --git a/MaskClustering/preprocess/scannet/prepare_gt.py b/MaskClustering/preprocess/scannet/prepare_gt.py new file mode 100644 index 0000000000000000000000000000000000000000..b63839e67d4c3183cc7d6d956d4ed3a33d9dff27 --- /dev/null +++ b/MaskClustering/preprocess/scannet/prepare_gt.py @@ -0,0 +1,95 @@ +from tqdm import tqdm +import os +import argparse +import json +from concurrent.futures import ProcessPoolExecutor +from itertools import repeat +import numpy as np +import pandas as pd +import sys +sys.path.append('../..') +from evaluation.constants import SCANNET_IDS + +raw_data_dir = '../../data/scannet/raw/scans' +gt_dir = '../../data/scannet/gt' +label_map_file = '../../data/scannet/raw/raw/scannetv2-labels.combined.tsv' +split_file_path = '../../splits/scannet.txt' + +CLOUD_FILE_PFIX = '_vh_clean_2' +SEGMENTS_FILE_PFIX = '.0.010000.segs.json' +AGGREGATIONS_FILE_PFIX = '.aggregation.json' + +def export_gt(filename, label_ids, instance_ids): + gt_data = label_ids * 1000 + instance_ids + 1 + np.savetxt(filename, gt_data, fmt='%d') + +# Map the raw category id to the point cloud +def point_indices_from_group(seg_indices, group, labels_pd): + group_segments = np.array(group['segments']) + label = group['label'] + + # Map the category name to id + label_ids = labels_pd[labels_pd['raw_category'] == label]['id'] + label_id = int(label_ids.iloc[0]) if len(label_ids) > 0 else 0 + + # Only store for the valid categories + if not label_id in SCANNET_IDS: + label_id = 0 + + # get points, where segment indices (points labelled with segment ids) are in the group segment list + point_IDs = np.where(np.isin(seg_indices, group_segments)) + + return point_IDs[0], label_id + +def handle_process(scene_path, output_path, labels_pd): + scene_id = scene_path.split('/')[-1] + segments_file = os.path.join(scene_path, f'{scene_id}{CLOUD_FILE_PFIX}{SEGMENTS_FILE_PFIX}') + aggregations_file = os.path.join(scene_path, f'{scene_id}{AGGREGATIONS_FILE_PFIX}') + + output_gt_file = os.path.join(output_path, f'{scene_id}.txt') + + # Load segments file + with open(segments_file) as f: + segments = json.load(f) + seg_indices = np.array(segments['segIndices']) + + # Load Aggregations file + with open(aggregations_file) as f: + aggregation = json.load(f) + seg_groups = np.array(aggregation['segGroups']) + + # Generate new labels + labelled_pc = np.zeros((len(seg_indices), 1)) + instance_ids = np.zeros((len(seg_indices), 1)) + for group in seg_groups: + p_inds, label_id = point_indices_from_group(seg_indices, group, labels_pd) + + labelled_pc[p_inds] = label_id + instance_ids[p_inds] = group['id'] + 1 + + labelled_pc = labelled_pc.astype(int) + instance_ids = instance_ids.astype(int) + + export_gt(output_gt_file, labelled_pc, instance_ids) + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--num_workers', default=16, type=int, help='The number of parallel workers') + config = parser.parse_args() + + # Load label map + labels_pd = pd.read_csv(label_map_file, sep='\t', header=0) + + with open(split_file_path) as val_file: + val_scenes = val_file.read().splitlines() + + # Load scene paths + scene_paths = [os.path.join(raw_data_dir, scene) for scene in val_scenes] + + os.makedirs(gt_dir, exist_ok=True) + + # Preprocess data. + pool = ProcessPoolExecutor(max_workers=config.num_workers) + print('Processing scenes...') + # show progress + _ = list(tqdm(pool.map(handle_process, scene_paths, repeat(gt_dir), repeat(labels_pd)), total=len(scene_paths))) \ No newline at end of file diff --git a/MaskClustering/preprocess/scannet/prepare_single_gt.py b/MaskClustering/preprocess/scannet/prepare_single_gt.py new file mode 100644 index 0000000000000000000000000000000000000000..4bc1b2fb796ec3fbcf0879be0efe9aa8d9e59da6 --- /dev/null +++ b/MaskClustering/preprocess/scannet/prepare_single_gt.py @@ -0,0 +1,76 @@ +import os +import argparse +import json +import numpy as np +import pandas as pd +import sys +sys.path.append('/home/jovyan/users/bulat/workspace/3drec/MaskClustering') +print("prepare_single_gt sys.path", sys.path) +from evaluation.constants import SCANNET_IDS + +def export_gt(filename, label_ids, instance_ids): + gt_data = label_ids * 1000 + instance_ids + 1 + np.savetxt(filename, gt_data, fmt='%d') + +def point_indices_from_group(seg_indices, group, labels_pd): + group_segments = np.array(group['segments']) + label = group['label'] + + # Map the category name to id + label_ids = labels_pd[labels_pd['raw_category'] == label]['id'] + label_id = int(label_ids.iloc[0]) if len(label_ids) > 0 else 0 + + # Only store for the valid categories + if not label_id in SCANNET_IDS: + label_id = 0 + + # get points, where segment indices (points labelled with segment ids) are in the group segment list + point_IDs = np.where(np.isin(seg_indices, group_segments)) + + return point_IDs[0], label_id + +def handle_single_scene(scene_path, output_path, labels_pd, scene_name): + segments_file = os.path.join(scene_path, f'{scene_name}_vh_clean_2.0.010000.segs.json') + aggregations_file = os.path.join(scene_path, f'{scene_name}.aggregation.json') + + output_gt_file = os.path.join(output_path, f'{scene_name}.txt') + + # Load segments file + with open(segments_file) as f: + segments = json.load(f) + seg_indices = np.array(segments['segIndices']) + + # Load Aggregations file + with open(aggregations_file) as f: + aggregation = json.load(f) + seg_groups = np.array(aggregation['segGroups']) + + # Generate new labels + labelled_pc = np.zeros((len(seg_indices), 1)) + instance_ids = np.zeros((len(seg_indices), 1)) + for group in seg_groups: + p_inds, label_id = point_indices_from_group(seg_indices, group, labels_pd) + + labelled_pc[p_inds] = label_id + instance_ids[p_inds] = group['id'] + 1 + + labelled_pc = labelled_pc.astype(int) + instance_ids = instance_ids.astype(int) + + export_gt(output_gt_file, labelled_pc, instance_ids) + print(f"Ground truth сохранен в {output_gt_file}") + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--scene_path', required=True, type=str, help='Путь к директории сцены') + parser.add_argument('--gt_dir', required=True, type=str, help='Директория для сохранения ground truth') + parser.add_argument('--label_map', required=True, type=str, help='Путь к файлу маппинга меток') + parser.add_argument('--scene_name', required=True, type=str, help='Имя сцены') + args = parser.parse_args() + + # Load label map + labels_pd = pd.read_csv(args.label_map, sep='\t', header=0) + + os.makedirs(args.gt_dir, exist_ok=True) + + handle_single_scene(args.scene_path, args.gt_dir, labels_pd, args.scene_name) diff --git a/MaskClustering/preprocess/scannet/process_val.py b/MaskClustering/preprocess/scannet/process_val.py new file mode 100644 index 0000000000000000000000000000000000000000..37b9db131d1757564bbe1dfeb0a4dbc8e5b09464 --- /dev/null +++ b/MaskClustering/preprocess/scannet/process_val.py @@ -0,0 +1,24 @@ +import os +from tqdm import tqdm +import shutil + +raw_data_dir = '../../data/scannet/raw/scans' +target_data_dir = '../../data/scannet/processed' +split_file_path = '../../splits/scannet.txt' + +def process_one_seq(seq_name): + target_seq_dir = os.path.join(target_data_dir, seq_name) + os.makedirs(target_data_dir, exist_ok=True) + + command = f'python reader.py --filename {raw_data_dir}/{seq_name}/{seq_name}.sens --output_path {target_seq_dir} --export_color_images --export_depth_images --export_poses --export_intrinsics' + os.system(command) + + # copy the point cloud file here + shutil.copyfile(f'{raw_data_dir}/{seq_name}/{seq_name}_vh_clean_2.ply', f'{target_seq_dir}/{seq_name}_vh_clean_2.ply') + +with open(split_file_path, 'r') as f: + seq_name_list = f.readlines() +seq_name_list = [seq_name.strip() for seq_name in seq_name_list] + +for seq_name in tqdm(seq_name_list): + process_one_seq(seq_name) \ No newline at end of file diff --git a/MaskClustering/preprocess/scannet/reader.py b/MaskClustering/preprocess/scannet/reader.py new file mode 100644 index 0000000000000000000000000000000000000000..a1e38e7a35b02fe89f18eaa63be2df854356d5aa --- /dev/null +++ b/MaskClustering/preprocess/scannet/reader.py @@ -0,0 +1,39 @@ +import argparse +import os, sys + +from SensorData import SensorData + +# params +parser = argparse.ArgumentParser() +# data paths +parser.add_argument('--filename', required=True, help='path to sens file to read') +parser.add_argument('--output_path', required=True, help='path to output folder') +parser.add_argument('--export_depth_images', dest='export_depth_images', action='store_true') +parser.add_argument('--export_color_images', dest='export_color_images', action='store_true') +parser.add_argument('--export_poses', dest='export_poses', action='store_true') +parser.add_argument('--export_intrinsics', dest='export_intrinsics', action='store_true') +parser.set_defaults(export_depth_images=False, export_color_images=False, export_poses=False, export_intrinsics=False) + +opt = parser.parse_args() +print(opt) + + +def main(): + if not os.path.exists(opt.output_path): + os.makedirs(opt.output_path) + # load the data + sys.stdout.write('loading %s...' % opt.filename) + sd = SensorData(opt.filename) + sys.stdout.write('loaded!\n') + if opt.export_depth_images: + sd.export_depth_images(os.path.join(opt.output_path, 'depth'), frame_skip=10) + if opt.export_color_images: + sd.export_color_images(os.path.join(opt.output_path, 'color'), frame_skip=10) + if opt.export_poses: + sd.export_poses(os.path.join(opt.output_path, 'pose'), frame_skip=10) + if opt.export_intrinsics: + sd.export_intrinsics(os.path.join(opt.output_path, 'intrinsic')) + + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/MaskClustering/preprocess/scannetpp/download_scannetpp.yml b/MaskClustering/preprocess/scannetpp/download_scannetpp.yml new file mode 100644 index 0000000000000000000000000000000000000000..415b9624a7119ea0705a554c79ac70eac30eeefa --- /dev/null +++ b/MaskClustering/preprocess/scannetpp/download_scannetpp.yml @@ -0,0 +1,207 @@ +################################################################################ +# _____ _ __ __ # +# / ___/ _____ ____ _ ____ / | / / ___ / /_ __ __ # +# \__ \ / ___/ / __ `/ / __ \ / |/ / / _ \ / __/ __/ /_ __/ /_ # +# ___/ / / /__ / /_/ / / / / / / /| / / __// /_ /_ __//_ __/ # +# /____/ \___/ \__,_/ /_/ /_/ /_/ |_/ \___/ \__/ /_/ /_/ # +# # +################################################################################ +# # +# TL;DR: How to download ScanNet++ data # +# ------------------------------------- # +# 1. Set your personalized token in `token`. # +# 2. Set the download location in `data_root`. # +# 3. (Optional) All scenes are downloaded by default. Select the splits to # +# download in `download_splits` or individual scenes by ID in # +# `download_scenes`. Scene lists can be downloaded by setting # +# `metadata_only` to true. # +# 4. (Optional) The assets in `default_assets` are downloaded by default. # +# Set `download_assets` or `download_options` to specify individual assets # +# or asset groups. # +# 6. Run: pip install -r requirements.txt # +# 5. Run: python download_scannetpp.py download_scannetpp.yml # +# # +################################################################################ + +###### token ###### enter your personalized token here +token: Your_token_here +###### download the data here ###### +# specify an absolute path +data_root: /raid/miyan/scannet++/ + +# root url of the data +root_url: https://kaldir.vc.in.tum.de/scannetpp/download?token=TOKEN&file=FILEPATH + +# download the metadata only, change this to true +metadata_only: false + +# show output for each file being download and unzipped +verbose: false + +###### specify the scenes to download ###### +# select the splits to download +# download_splits: [nvs_sem_train, nvs_sem_val, nvs_test, sem_test] +download_splits: [nvs_sem_val] + + +# uncomment this and comment above to download data only for specific scenes +# enter the scene IDs separated by comma +# scene lists can be downloaded by setting metadata_only to true +# an example scene ID is provided below +# download_scenes: [0d2ee665be] + +###### specify the assets to download ###### +# by default, these assets are downloaded for each scene: +# mesh+3D semantics, lowres dslr, iphone data +# see `scene_assets` and the dataset documentation page for more info on each asset +# default_assets: [ +# # dslr lists, images +# dslr_train_test_lists_path, dslr_resized_dir, dslr_resized_mask_dir, +# # camera poses +# dslr_colmap_dir, dslr_nerfstudio_transform_path, +# # mesh +# scan_mesh_path, scan_mesh_mask_path, +# # annotation +# scan_mesh_segs_path, scan_anno_json_path, scan_sem_mesh_path, +# # iphone video and depth +# iphone_video_path, iphone_video_mask_path, iphone_depth_path, +# # camera poses +# iphone_pose_intrinsic_imu_path, iphone_colmap_dir, iphone_exif_path +# ] + +default_assets: [ + # mesh + scan_mesh_path, scan_mesh_mask_path, + # annotation + scan_mesh_segs_path, scan_anno_json_path, scan_sem_mesh_path, + # iphone video and depth + iphone_video_path, iphone_video_mask_path, iphone_depth_path, + # camera poses + iphone_pose_intrinsic_imu_path, iphone_colmap_dir, iphone_exif_path + ] + +# uncomment this and comment the other asset options to specify individual assets +# the full list of assets is under `scene_assets` +# download_assets: [] + +# uncomment this and comment the other asset options to specify options based on device or task +# see below for possible options +# download_options: [] + +############################################################## +############## Nothing to be changed below this ############## +############################################################## +# Options by task: +# - nvs_dslr: novel view synthesis with DSLR images +# - nvs_iphone: novel view synthesis with iPhone images +# - semantic: download RGB mesh and semantic annotations + +# Options by device: +# - scans: all scan related data - point cloud, mesh +# - dslr_hires: all dslr data including hires +# - iphone: all iphone RGBD data +option_assets: + nvs_dslr: [dslr_train_test_lists_path, dslr_resized_dir, dslr_resized_mask_dir, + dslr_colmap_dir, dslr_nerfstudio_transform_path] + nvs_iphone: [iphone_video_path, iphone_video_mask_path, iphone_pose_intrinsic_imu_path, + iphone_colmap_dir] + semantic: [scan_mesh_path, scan_mesh_segs_path, scan_anno_json_path, scan_sem_mesh_path] + scans: [scan_transformed_poses_path, scan_pc_path, scan_pc_mask_path, + scan_mesh_path, scan_mesh_mask_path, + scan_mesh_segs_path, scan_anno_json_path, scan_sem_mesh_path] + dslr_hires: [dslr_train_test_lists_path, dslr_resized_dir, dslr_resized_mask_dir, + dslr_original_dir, dslr_original_mask_dir, + dslr_colmap_dir, dslr_nerfstudio_transform_path] + iphone: [iphone_video_path, iphone_video_mask_path, iphone_depth_path, + iphone_pose_intrinsic_imu_path, iphone_colmap_dir] + +# splits in the dataset +splits: [nvs_sem_train, nvs_sem_val, nvs_test, sem_test] + +# other meta files to download +meta_files: + - splits/nvs_sem_train.txt + - splits/nvs_sem_val.txt + - splits/nvs_test.txt + - splits/sem_test.txt + - metadata/semantic_classes.txt + - metadata/instance_classes.txt + - metadata/semantic_benchmark/top100.txt + - metadata/semantic_benchmark/top100_instance.txt + - metadata/semantic_benchmark/map_benchmark.csv + +#### all paths and directories to download for each scene +scene_assets: + #### dslr #### + # images + # resized dslr images + - dslr_resized_dir + # anonymization masks for dslr resized dslr images + - dslr_resized_mask_dir + # original dslr images + - dslr_original_dir + # anonymization masks for original dslr images + - dslr_original_mask_dir + ## camera poses + # colmap models + - dslr_colmap_dir + # in nerfstudio format + - dslr_nerfstudio_transform_path + # train and test image lists + - dslr_train_test_lists_path + ##### scan ##### + ## scans + # point cloud + - scan_pc_path + # mask of anonymized points + - scan_pc_mask_path + # scanner positions + - scan_transformed_poses_path + ## mesh + # RGB mesh + - scan_mesh_path + # mask of anonymized mesh vertices + - scan_mesh_mask_path + ## 3d semantic annotation on mesh + # mesh surface segments + - scan_mesh_segs_path + # annotation on mesh segments + - scan_anno_json_path + # mesh with semantic labels on vertices + - scan_sem_mesh_path + # #### iphone #### + ## rgb + # rgb video + - iphone_video_path + # video of anonymization masks for rgb video + - iphone_video_mask_path + # depth, binary file + - iphone_depth_path + # ARKit metadata + - iphone_pose_intrinsic_imu_path + ## camera poses, similar to DSLR + - iphone_colmap_dir + - iphone_nerfstudio_transform_path + # exif data + - iphone_exif_path + +# some assets are not present in the test sets +exclude_assets: + # 3D information and dslr test images + # not present in the nvs test set + nvs_test: [iphone_depth_path, + scan_pc_path, scan_pc_mask_path, scan_transformed_poses_path, + scan_mesh_path, scan_mesh_mask_path, + scan_mesh_segs_path, scan_anno_json_path, scan_sem_mesh_path] + # all annotation related files not present in the sem test set + sem_test: [scan_mesh_segs_path, scan_anno_json_path, scan_sem_mesh_path] + +# unzip these assets +zipped_assets: [ + dslr_resized_dir, dslr_resized_mask_dir, dslr_original_dir, dslr_original_mask_dir, + dslr_colmap_dir, + scan_pc_path, scan_mesh_path, scan_sem_mesh_path, + scan_mesh_segs_path, scan_anno_json_path, + iphone_video_mask_path, iphone_depth_path, iphone_pose_intrinsic_imu_path, + iphone_colmap_dir, iphone_nerfstudio_transform_path, iphone_exif_path +] \ No newline at end of file diff --git a/MaskClustering/preprocess/scannetpp/prepare_iphone_data.yml b/MaskClustering/preprocess/scannetpp/prepare_iphone_data.yml new file mode 100644 index 0000000000000000000000000000000000000000..e399d6de6747abfea9aa8a2c72cf4cc0e2b6711e --- /dev/null +++ b/MaskClustering/preprocess/scannetpp/prepare_iphone_data.yml @@ -0,0 +1,12 @@ + +# extract RGB frames +extract_rgb: True +# extract anonymization masks +extract_masks: false +# extract depth frames +extract_depth: false + +# folder where the data is downloaded +data_root: data + +splits: [nvs_sem_val] \ No newline at end of file diff --git a/MaskClustering/preprocess/scannetpp/prepare_semantic_gt.yml b/MaskClustering/preprocess/scannetpp/prepare_semantic_gt.yml new file mode 100644 index 0000000000000000000000000000000000000000..ed9feffdd0fd5ed8ffc8e7e844bf8c63af4a18c1 --- /dev/null +++ b/MaskClustering/preprocess/scannetpp/prepare_semantic_gt.yml @@ -0,0 +1,22 @@ +pth_dir: data/pcld_0.25 +scene_list: data/splits/nvs_sem_val.txt + +# save in npy format? +save_npy: false +# save in txt format? +save_txt: true + +## SEMANTIC +save_semantic: false +sem_out_dir: + +## INSTANCE +save_instance: true +# save in gt format: sem*1000 + inst id +# save in preds format: 1 file for list of instances, 1 file for each instance +# same as scannet +inst_gt_format: true +inst_gtformat_out_dir: data/gt + +inst_preds_format: false +inst_predsformat_out_dir: diff --git a/MaskClustering/preprocess/scannetpp/prepare_training_data.yml b/MaskClustering/preprocess/scannetpp/prepare_training_data.yml new file mode 100644 index 0000000000000000000000000000000000000000..921f5195e58b929b81201155a9242271a8686a99 --- /dev/null +++ b/MaskClustering/preprocess/scannetpp/prepare_training_data.yml @@ -0,0 +1,33 @@ +data: + data_root: data/data + + labels_path: data/metadata/semantic_classes.txt + # for instance segmentation + use_instances: true + instance_labels_path: data/metadata/instance_classes.txt + + ## save multiple labels per vertex/point? ## + # multilabel: + # max_gt: 3 + # multilabel_only: false + + mapping_file: data/metadata/semantic_benchmark/map_benchmark.csv + + list_path: data/splits/nvs_sem_val.txt + + ignore_label: -100 + + sample_factor: 0.25 + + transforms: + # read the mesh + - add_mesh_vertices + # map raw labels to benchmark classes + - map_label_to_index + # use segments info to get labels on the vertices, handle multilabels + - get_labels_on_vertices + # # sample points on the mesh and transfer all vertex info to the points + - sample_points_on_mesh + +# dir to save pth training data +out_dir: data/pcld_0.25 diff --git a/MaskClustering/preprocess/scannetpp/render.yml b/MaskClustering/preprocess/scannetpp/render.yml new file mode 100644 index 0000000000000000000000000000000000000000..c7f64ea15e23ec7c65146c8aa543e2278b5fd188 --- /dev/null +++ b/MaskClustering/preprocess/scannetpp/render.yml @@ -0,0 +1,19 @@ +# folder where the data is downloaded +data_root: data/ + +# Set True to render depth for iphone frames +render_iphone: True +# Set True to render depth for dslr frames +render_dslr: False + +splits: [nvs_sem_val] + +# Specify scene ids if you want to render depth for specific scenes +# scene_ids: [355e5e32db] + +# The near and far planes for the depth camera during rendering in meters. +near: 0.05 +far: 20.0 + +# Output directory for the rendered depth images. If not given, the output will be saved to data folder in data_root +output_dir: data/data diff --git a/MaskClustering/proc_masks.py b/MaskClustering/proc_masks.py new file mode 100644 index 0000000000000000000000000000000000000000..0176c093f23b5dd322631158e65e956c59ccddc1 --- /dev/null +++ b/MaskClustering/proc_masks.py @@ -0,0 +1,201 @@ +import numpy as np +import os +import cv2 +from pathlib import Path +import trimesh as tm +from sklearn.neighbors import KDTree +from tqdm import tqdm +from tqdm.contrib.concurrent import thread_map + +def process_frame(frame, vertices, intrinsics, source_path, base_path, key): + frame_id = str(frame['frame_id']).zfill(5) + mask_path = frame['mask_path'] + mask_path = base_path / mask_path + mask = np.load(mask_path, allow_pickle=True) + mask = mask == key + depth = cv2.imread(source_path / f'{frame_id}.png', cv2.IMREAD_UNCHANGED) / 1000. + + extrinsics = np.loadtxt(source_path / f'{frame_id}.txt') + + point_mask = np.zeros(len(vertices), dtype=bool) + + + kernel_size = 3 + post_process_erosion = True + post_process_dilation = False + post_process_component = True + post_process_component_num = 1 + + + img = np.uint8(mask) * 255 + + # Define the kernel for morphological operations using cv.getStructuringElement + # Поддержка различных форм ядер: MORPH_RECT, MORPH_CROSS, MORPH_ELLIPSE + kernel_shape = cv2.MORPH_ELLIPSE # Эллиптическая форма для более плавной эрозии + kernel = cv2.getStructuringElement(kernel_shape, + (2 * kernel_size + 1, 2 * kernel_size + 1), + (kernel_size, kernel_size)) + + # Apply morphological erosion if requested + if post_process_erosion: + # Увеличиваем количество итераций эрозии для более сильного уменьшения + img = cv2.erode(img, kernel, iterations=1) + + # Apply morphological dilation if requested + if post_process_dilation: + # Уменьшаем дилатацию, чтобы не компенсировать эрозию полностью + img = cv2.dilate(img, kernel, iterations=1) + + # Find all connected components + num_labels, labels_im = cv2.connectedComponents( + img + ) # label 0 is background, so start from 1 + if post_process_component and num_labels > 1: + # Calculate the area of each component and sort them, keeping the largest k + component_areas = [ + (label, np.sum(labels_im == label)) for label in range(1, num_labels) + ] + component_areas.sort(key=lambda x: x[1], reverse=True) + largest_components = [ + x[0] for x in component_areas[: post_process_component_num] + ] + img = np.isin(labels_im, largest_components).astype(np.uint8) + + # Return the processed image as a boolean mask + + + # cv2.imwrite("new_mask.png", img * 255) + mask = cv2.resize(img, depth.shape[::-1]) + mask = mask > 0.5 + mask = mask & (depth > 0) + + cv2.imwrite("mask.png", (mask * 255).astype(np.uint8)) + + # cv2.imwrite("new_mask_wd.png", (mask).astype(np.uint8) * 255) + depth_y, depth_x = np.where(mask) + depths = depth[mask] + + + + + if len(depth_x) == 0: + return np.zeros(len(vertices), dtype=bool) + + # Создаем однородные координаты пикселей + pixel_coords = np.vstack([depth_x, depth_y, np.ones(len(depth_x))]) + + + # Шаг 1: Обратная проекция пикселей в нормализованные координаты камеры + normalized_coords = np.linalg.inv(intrinsics) @ pixel_coords + + # Шаг 2: Масштабируем нормализованные координаты на глубину для получения 3D точек в системе камеры + camera_points_3d = normalized_coords * depths[np.newaxis, :] + + # Шаг 3: Добавляем однородную координату для трансформации в мировые координаты + camera_points_homogeneous = np.vstack([camera_points_3d, np.ones(len(depth_x))]) + + # Шаг 4: Трансформируем из координат камеры в мировые координаты + # Используем прямую трансформацию extrinsics (camera-to-world) + world_points_homogeneous = extrinsics @ camera_points_homogeneous + + # Шаг 5: Нормализуем однородные координаты + points = (world_points_homogeneous[:3, :] / world_points_homogeneous[3, :]).T + + points = points[~np.isnan(points).any(axis=1)] + if len(points) == 0: + return np.zeros(len(vertices), dtype=bool) + tree = KDTree(vertices) + + dist, ind = tree.query(points, k=1) + ind = ind.flatten() + dist = dist.flatten() + + max_distance = 0.05 # 10 см максимальное расстояние + valid_matches = dist < max_distance + ind = ind[valid_matches] + ind = np.unique(ind) + print(f"unique ind: {len(ind)}") + + + if valid_matches.sum() > 0: + point_mask[ind] = True + + return point_mask + +def process_object(data): + key, item, vertices, intrinsics, source_path, base_path, num_frames = data + frames = item['frames'] + total_points_mask = np.zeros(len(vertices), dtype=bool) + for frame in frames[:num_frames]: + point_mask = process_frame(frame, vertices, intrinsics, source_path, base_path, key) + total_points_mask = total_points_mask | point_mask + return total_points_mask + + +def load_scan(pcd_path): + pcd_data = np.fromfile(pcd_path, dtype=np.float32).reshape(-1, 6)[:, :3] + return pcd_data + +def process_scene(data): + scene_id, exp_name = data + pred_path = Path(f"data/prediction/scannet/baseline_scannet200/{scene_id}.npz") + out_path = Path(f"data/prediction/scannet/{exp_name}/{scene_id}.npz") + base_path = Path(f"/home/jovyan/users/lemeshko/scripts/gsam_result/yolo/{scene_id}") + source_path = Path(f"/home/jovyan/users/kolodiazhnyi/data/scannet/posed_images/{scene_id}") + scan_path = Path(f"/home/jovyan/users/bulat/workspace/3drec/Indoor/OKNO/data/scannet200/points/{scene_id}.bin") + info_path = base_path / "infos.npy" + + # if out_path.exists(): + # return + vertices = load_scan(scan_path) + info_data = np.load(info_path, allow_pickle=True).item() + + base_data = np.load(pred_path, allow_pickle=True) + + # Диагностика меша + print(f"Mesh vertices shape: {vertices.shape}") + print(f"Mesh vertices range:") + print(f" X: [{vertices[:, 0].min():.3f}, {vertices[:, 0].max():.3f}]") + print(f" Y: [{vertices[:, 1].min():.3f}, {vertices[:, 1].max():.3f}]") + print(f" Z: [{vertices[:, 2].min():.3f}, {vertices[:, 2].max():.3f}]") + + + intrinsics = np.loadtxt(source_path / 'intrinsic.txt')[:3, :3] + intrinsics[0, :] *= 640 / 1296 + intrinsics[1, :] *= 480 / 968 + + num_frames = 500 + object_data = [[key, item, vertices, intrinsics, source_path, base_path, num_frames] for key, item in info_data.items()] + total_points_masks = thread_map(process_object, object_data, chunksize=100) + + + new_data = { + k: v for k, v in base_data.items() + } + for i, key in enumerate(info_data.keys()): + new_data['pred_masks'][:, i] = total_points_masks[i] + out_path.parent.mkdir(parents=True, exist_ok=True) + vs = [] + cs = [] + for i in range(new_data['pred_masks'].shape[1]): + os.makedirs(f"pred_masks", exist_ok=True) + v = vertices[new_data['pred_masks'][:, i]] + c = np.random.rand(3) + c = np.repeat(c[np.newaxis, :], len(v), axis=0) + vs.append(v) + cs.append(c) + tm.PointCloud(np.concatenate(vs, axis=0), colors=np.concatenate(cs, axis=0)).export(f"pred_masks/{scene_id}_mask.ply") + + print("uniques", np.unique(new_data['pred_masks'].sum(1)), [[k, v.shape] for k, v in new_data.items()]) + np.savez(out_path, **new_data) + + + +if __name__ == "__main__": + exp_name = "erode_mask" + scenes = np.loadtxt("/home/jovyan/users/bulat/workspace/3drec/Indoor/MaskClustering/splits/scannet200_subset.txt", dtype=str) + for scene in scenes: + process_scene((scene, exp_name)) + + + diff --git a/MaskClustering/proc_masks_fixed.py b/MaskClustering/proc_masks_fixed.py new file mode 100644 index 0000000000000000000000000000000000000000..18b7b62fb4a9c1fedda03656a4ea068f07cb5b20 --- /dev/null +++ b/MaskClustering/proc_masks_fixed.py @@ -0,0 +1,192 @@ +import numpy as np +import os +import cv2 +from pathlib import Path +import trimesh as tm +from sklearn.neighbors import KDTree + +def unproject_depth_to_world(depth_x, depth_y, depths, intrinsics, extrinsics): + """ + Правильная функция для обратной проекции пикселей в мировые координаты + + Args: + depth_x, depth_y: координаты пикселей + depths: значения глубины в метрах + intrinsics: внутренние параметры камеры (3x3) + extrinsics: внешние параметры камеры (4x4, world-to-camera) + + Returns: + points: мировые координаты точек (Nx3) + """ + # 1. Создаем однородные координаты пикселей + pixel_coords = np.vstack([depth_x, depth_y, np.ones(len(depth_x))]) + + # 2. Обратная проекция в координаты камеры + # K^-1 * [u, v, 1]^T дает нормализованные координаты + camera_rays = np.linalg.inv(intrinsics) @ pixel_coords + + # 3. Масштабируем на глубину для получения 3D точек в системе камеры + camera_points = camera_rays * depths[np.newaxis, :] + + # 4. Добавляем однородную координату + camera_points_homogeneous = np.vstack([camera_points, np.ones(len(depth_x))]) + + # 5. Преобразуем в мировые координаты + # Extrinsics - это world-to-camera, нам нужна обратная матрица + world_points_homogeneous = np.linalg.inv(extrinsics) @ camera_points_homogeneous + + # 6. Нормализуем однородные координаты + world_points = world_points_homogeneous[:3, :] / world_points_homogeneous[3, :] + + return world_points.T + +def process_mask_with_morphology(mask, kernel_size=5, + post_process_erosion=True, + post_process_dilation=True, + post_process_component=True, + post_process_component_num=1): + """ + Обработка маски с морфологическими операциями + """ + img = np.uint8(mask) * 255 + + # Определяем ядро для морфологических операций + kernel = np.ones((kernel_size * 2 + 1, kernel_size * 2 + 1), np.uint8) + + # Применяем эрозию + if post_process_erosion: + img = cv2.erode(img, kernel, iterations=1) + + # Применяем дилатацию + if post_process_dilation: + img = cv2.dilate(img, kernel, iterations=1) + + # Находим связанные компоненты + num_labels, labels_im = cv2.connectedComponents(img) + + if post_process_component and num_labels > 1: + # Вычисляем площадь каждой компоненты и сортируем + component_areas = [ + (label, np.sum(labels_im == label)) for label in range(1, num_labels) + ] + component_areas.sort(key=lambda x: x[1], reverse=True) + largest_components = [ + x[0] for x in component_areas[:post_process_component_num] + ] + img = np.isin(labels_im, largest_components).astype(np.uint8) + + return img.astype(bool) + +if __name__ == "__main__": + scene_id = "scene0011_00" + path = Path(f"/home/jovyan/users/bulat/workspace/3drec/Indoor/MaskClustering/data/prediction/scannet/test/{scene_id}.npz") + base_path = Path(f"/home/jovyan/users/lemeshko/scripts/gsam_result/yolo/{scene_id}") + source_path = Path(f"/home/jovyan/users/kolodiazhnyi/data/scannet/posed_images/{scene_id}") + scan_path = Path(f"data/scannet/processed/{scene_id}/{scene_id}_vh_clean_2.ply") + info_path = base_path / "infos.npy" + + # Проверяем существование файлов + if not scan_path.exists(): + raise FileNotFoundError(f"Mesh file not found: {scan_path}") + if not info_path.exists(): + raise FileNotFoundError(f"Info file not found: {info_path}") + + # Загружаем данные + mesh = tm.load(scan_path) + vertices = mesh.vertices + data = np.load(path, allow_pickle=True) + info_data = np.load(info_path, allow_pickle=True).item() + + key, item = next(iter(info_data.items())) + print(f"Processing object {key}") + print(f"Object info: {item}") + + frames = item['frames'] + intrinsics = np.loadtxt(source_path / 'intrinsic.txt') + + frame = frames[0] + frame_id = str(frame['frame_id']).zfill(5) + mask_path = frame['mask_path'] + mask_path = base_path / mask_path + + # Проверяем существование файлов кадра + depth_file = source_path / f'{frame_id}.png' + extrinsics_file = source_path / f'{frame_id}.txt' + + if not depth_file.exists(): + raise FileNotFoundError(f"Depth file not found: {depth_file}") + if not extrinsics_file.exists(): + raise FileNotFoundError(f"Extrinsics file not found: {extrinsics_file}") + + # Загружаем маску, глубину и экстринсики + mask = np.load(mask_path, allow_pickle=True) + mask = mask == key + depth = cv2.imread(str(depth_file), -1) / 1000.0 # Конвертируем в метры + extrinsics = np.loadtxt(extrinsics_file) + + print(f"Original mask shape: {mask.shape}") + print(f"Depth shape: {depth.shape}") + print(f"Mask pixels count: {mask.sum()}") + + # Обрабатываем маску морфологическими операциями + processed_mask = process_mask_with_morphology( + mask, + kernel_size=5, + post_process_erosion=True, + post_process_dilation=True, + post_process_component=True, + post_process_component_num=1 + ) + + # Изменяем размер маски под размер карты глубины + final_mask = cv2.resize( + processed_mask.astype(np.uint8), + depth.shape[::-1], + interpolation=cv2.INTER_NEAREST_EXACT + ).astype(bool) + + print(f"Final mask shape: {final_mask.shape}") + print(f"Final mask pixels count: {final_mask.sum()}") + + # Находим координаты пикселей с маской + depth_y, depth_x = np.where(final_mask) + depths = depth[final_mask] + + # Фильтруем точки с недействительной глубиной + valid_depth = (depths > 0) & (depths < 10.0) # Разумные пределы глубины + depth_x = depth_x[valid_depth] + depth_y = depth_y[valid_depth] + depths = depths[valid_depth] + + print(f"Valid depth points: {len(depths)}") + + if len(depths) == 0: + print("No valid depth points found!") + else: + # ИСПРАВЛЕННАЯ проекция в мировые координаты + world_points = unproject_depth_to_world(depth_x, depth_y, depths, intrinsics, extrinsics) + + print(f"World points shape: {world_points.shape}") + print(f"World points range:") + print(f" X: [{world_points[:, 0].min():.3f}, {world_points[:, 0].max():.3f}]") + print(f" Y: [{world_points[:, 1].min():.3f}, {world_points[:, 1].max():.3f}]") + print(f" Z: [{world_points[:, 2].min():.3f}, {world_points[:, 2].max():.3f}]") + + # Находим ближайшие вершины с ограничением по расстоянию + tree = KDTree(vertices) + distances, indices = tree.query(world_points, k=1) + + # Фильтруем по максимальному расстоянию (например, 0.05 метра) + max_distance = 0.05 + valid_matches = distances.flatten() < max_distance + + print(f"Points within {max_distance}m: {valid_matches.sum()}/{len(valid_matches)}") + + # Создаем маску точек + point_mask = np.zeros(len(vertices), dtype=bool) + if valid_matches.sum() > 0: + point_mask[indices.flatten()[valid_matches]] = True + + print(f"Final point mask sum: {point_mask.sum()}") + print(f"Mesh vertices total: {len(vertices)}") + print(f"Coverage: {point_mask.sum()/len(vertices)*100:.2f}%") \ No newline at end of file diff --git a/MaskClustering/run.py b/MaskClustering/run.py new file mode 100644 index 0000000000000000000000000000000000000000..f6397691b53af4c0a4a4a7b53a0e92b20503094b --- /dev/null +++ b/MaskClustering/run.py @@ -0,0 +1,105 @@ +import os +from tqdm import tqdm +import time +from utils.config import get_args + +CUDA_LIST = [0] + +def execute_commands(commands_list, command_type, process_num): + print('====> Start', command_type) + from multiprocessing import Pool + pool = Pool(process_num) + for _ in tqdm(pool.imap_unordered(os.system, commands_list), total=len(commands_list)): + pass + pool.close() + pool.join() + pool.terminate() + print('====> Finish', command_type) + +def get_seq_name_list(dataset): + if dataset == 'scannet': + file_path = 'splits/scannet.txt' + elif dataset == 'scannetpp': + file_path = 'splits/scannetpp.txt' + elif dataset == 'matterport3d': + file_path = 'splits/matterport3d.txt' + with open(file_path, 'r') as f: + seq_name_list = f.readlines() + seq_name_list = [seq_name.strip() for seq_name in seq_name_list] + return seq_name_list + +def parallel_compute(general_command, command_name, resource_type, cuda_list, seq_name_list): + cuda_num = len(cuda_list) + + if resource_type == 'cuda': + commands = [] + for i, cuda_id in enumerate(cuda_list): + process_seq_name = seq_name_list[i::cuda_num] + if len(process_seq_name) == 0: + continue + process_seq_name = '+'.join(process_seq_name) + command = f'CUDA_VISIBLE_DEVICES={cuda_id} {general_command % process_seq_name}' + commands.append(command) + execute_commands(commands, command_name, cuda_num) + elif resource_type == 'cpu': + commands = [] + for seq_name in seq_name_list: + commands.append(f'{general_command} --seq_name {seq_name}') + execute_commands(commands, command_name, cuda_num) + +def get_label_text_feature(cuda_id): + label_text_feature_path = 'data/text_features/matterport3d.npy' + if os.path.exists(label_text_feature_path): + return + command = f'CUDA_VISIBLE_DEVICES={cuda_id} python -m semantics.extract_label_featrues' + os.system(command) + +def main(args): + dataset = args.dataset + config = args.config + cropformer_path = args.cropformer_path + + if dataset == 'scannet': + root = 'data/scannet_dust3r_unposed/processed' + image_path_pattern = 'color/*.jpg' # stride = 10 + gt = 'data/scannet_dust3r_posed/gt' + elif dataset == 'scannetpp': + root = 'data/scannetpp/data' + image_path_pattern = 'iphone/rgb/*0.jpg' + gt = 'data/scannetpp/gt' + elif dataset == 'matterport3d': + root = 'data/matterport3d/scans' + image_path_pattern = '*/undistorted_color_images/*.jpg' # stride = 1 + gt = 'data/matterport3d/gt' + + t0 = time.time() + seq_name_list = os.listdir(root) + # print('There are %d scenes' % len(seq_name_list)) + + # Step 1: use Cropformer to get 2D instance masks for all sequences. + # parallel_compute(f'python third_party/detectron2/projects/CropFormer/demo_cropformer/mask_predict.py --config-file third_party/detectron2/projects/CropFormer/configs/entityv2/entity_segmentation/mask2former_hornet_3x.yaml --root {root} --image_path_pattern {image_path_pattern} --dataset {args.dataset} --seq_name_list %s --opts MODEL.WEIGHTS {cropformer_path}', 'predict mask', 'cuda', CUDA_LIST, seq_name_list) + + # # Step 2: Mask clustering using our proposed method. + # parallel_compute(f'python main.py --config {config} --seq_name_list %s', 'mask clustering', 'cuda', CUDA_LIST, seq_name_list) + + # Step 3: Evaluate the class-agnostic results. + # os.system(f'python -m evaluation.evaluate --pred_path data/prediction/{config}_class_agnostic --gt_path {gt} --dataset {dataset} --no_class') + + # Step 4: Get the open-vocabulary semantic features for each 2D masks. + # parallel_compute(f'python -m semantics.get_open-voc_features --config {config} --seq_name_list %s', 'get open-vocabulary semantic features using CLIP', 'cuda', CUDA_LIST, seq_name_list) + + # Step 5: Get the text CLIP features for each label. + # get_label_text_feature(CUDA_LIST[0]) + + # Step 6: Get labels for each 3D instances. + parallel_compute(f'python -m semantics.open-voc_query --config {config}', 'get text labels', 'cpu', CUDA_LIST, seq_name_list) + + # Step 7: Evaluate the class-aware results. + # os.system(f'python -m evaluation.evaluate --pred_path data/prediction/{config} --gt_path {gt} --dataset {dataset}') + + print('total time', (time.time() - t0)//60, 'min') + print('Average time', (time.time() - t0) / len(seq_name_list), 'sec') + +if __name__ == '__main__': + args = get_args() + main(args) \ No newline at end of file diff --git a/MaskClustering/run_arkit.py b/MaskClustering/run_arkit.py new file mode 100644 index 0000000000000000000000000000000000000000..14b7a1943538feedc0e95abb8a6c7c934f21ebed --- /dev/null +++ b/MaskClustering/run_arkit.py @@ -0,0 +1,125 @@ +import os +from tqdm import tqdm +import time +from utils.config import get_args +from pathlib import Path +import pickle + + +def execute_commands(commands_list, command_type, process_num): + print('====> Start', command_type) + from multiprocessing import Pool + pool = Pool(process_num) + for _ in tqdm(pool.imap_unordered(os.system, commands_list), total=len(commands_list)): + pass + pool.close() + pool.join() + pool.terminate() + print('====> Finish', command_type) + +def get_seq_name_list(dataset): + if dataset == 'scannet': + file_path = 'splits/scannet.txt' + elif dataset == 'scannetpp': + file_path = 'splits/scannetpp.txt' + elif dataset == 'scannetpp_dust3r_posed': + file_path = 'splits/scannetpp.txt' + elif dataset == 'scannetpp_dust3r_filtered_depth': + file_path = 'splits/scannetpp.txt' + elif dataset == 'scannetpp_mapanything_posed': + file_path = 'splits/scannetpp.txt' + elif dataset == 'scannetpp_dust3r_unposed': + file_path = 'splits/scannetpp.txt' + elif dataset == 'matterport3d': + file_path = 'splits/matterport3d.txt' + with open(file_path, 'r') as f: + seq_name_list = f.readlines() + seq_name_list = [seq_name.strip() for seq_name in seq_name_list] + return seq_name_list + +def parallel_compute(general_command, command_name, resource_type, cuda_list, seq_name_list): + cuda_num = len(cuda_list) + + if resource_type == 'cuda': + commands = [] + for i, cuda_id in enumerate(cuda_list): + process_seq_name = seq_name_list[i::cuda_num] + if len(process_seq_name) == 0: + continue + process_seq_name = '+'.join(process_seq_name) + command = f'CUDA_VISIBLE_DEVICES={cuda_id} {general_command % process_seq_name}' + commands.append(command) + execute_commands(commands, command_name, cuda_num) + elif resource_type == 'cpu': + commands = [] + for seq_name in seq_name_list: + commands.append(f'{general_command} --seq_name {seq_name}') + execute_commands(commands, command_name, cuda_num) + +def get_label_text_feature(cuda_id): + label_text_feature_path = 'data/text_features/matterport3d.npy' + if os.path.exists(label_text_feature_path): + return + command = f'CUDA_VISIBLE_DEVICES={cuda_id} python -m semantics.extract_label_featrues' + os.system(command) + +def main(args): + CUDA_LIST = args.devices + dataset = "scannetpp" + config = args.config + cropformer_path = args.cropformer_path + + root = f'data/{config}/processed' + image_path_pattern = 'color/*.jpg' # stride = 10 + # gt = 'data/scannet_dust3r_posed/gt' + + t0 = time.time() + # seq_name_list = get_seq_name_list(config) + seq_name_list = os.listdir(root) + + # val_path = Path("../") / "OKNO/data/arkitscenes/arkitscenes_offline_infos_val.pkl" + # out_dir = Path("data/arkit_vggt/processed") + # with open(val_path, "rb") as f: + # data_list = pickle.load(f)['data_list'] + + # val_scenes = [scene["lidar_points"]["lidar_path"] for scene in data_list][:2500] + # def extract_name(item): + # return item.split("_")[0] + # val_scenes = set([extract_name(scene) for scene in val_scenes]) + # seq_name_list = list(seq_name_list.intersection(val_scenes)) + # seq_name_list = ["scene0011_00", "scene0011_01", "scene0015_00", "scene0019_00", "scene0019_01", "scene0025_00", "scene0025_01", "scene0025_02", "scene0030_00", "scene0030_01", "scene0030_02", "scene0046_00", "scene0046_01", "scene0046_02", "scene0050_00", "scene0050_01", "scene0050_02", "scene0063_00", "scene0064_00", "scene0064_01", "scene0077_00", "scene0077_01", "scene0081_00", "scene0081_01", "scene0081_02", "scene0084_00", "scene0084_01", "scene0084_02", "scene0086_00", "scene0086_01", "scene0086_02", "scene0088_00", "scene0088_01", "scene0088_02", "scene0088_03", "scene0095_00", "scene0095_01", "scene0100_00", "scene0100_01", "scene0100_02", "scene0131_00", "scene0131_01", "scene0131_02", "scene0139_00", "scene0144_00", "scene0144_01", "scene0146_00", "scene0146_01", "scene0146_02", "scene0149_00", "scene0153_00", "scene0153_01", "scene0164_00", "scene0164_01", "scene0164_02", "scene0164_03", "scene0169_00", "scene0169_01", "scene0187_00", "scene0187_01", "scene0193_00", "scene0193_01", "scene0196_00", "scene0203_00", "scene0203_01", "scene0203_02", "scene0207_00", "scene0207_01", "scene0207_02", "scene0208_00", "scene0217_00", "scene0221_00", "scene0221_01", "scene0222_00", "scene0222_01", "scene0231_00", "scene0231_01", "scene0231_02", "scene0246_00", "scene0249_00", "scene0251_00", "scene0256_00", "scene0256_01", "scene0256_02", "scene0257_00", "scene0277_00", "scene0277_01", "scene0277_02", "scene0278_00", "scene0278_01", "scene0300_00", "scene0300_01", "scene0304_00", "scene0307_00", "scene0307_01", "scene0307_02", "scene0314_00", "scene0316_00", "scene0328_00", "scene0329_00", "scene0329_01", "scene0329_02", "scene0334_00", "scene0334_01", "scene0334_02", "scene0338_00", "scene0338_01", "scene0338_02", "scene0342_00", "scene0343_00", "scene0351_00", "scene0351_01", "scene0353_00", "scene0353_01", "scene0353_02", "scene0354_00", "scene0355_00", "scene0355_01", "scene0356_00", "scene0356_01", "scene0356_02", "scene0357_00", "scene0357_01", "scene0377_00", "scene0377_01", "scene0377_02", "scene0378_00", "scene0378_01", "scene0378_02", "scene0382_00", "scene0382_01", "scene0389_00", "scene0406_00", "scene0406_01", "scene0406_02", "scene0412_00", "scene0412_01", "scene0414_00", "scene0423_00", "scene0423_01", "scene0423_02", "scene0426_00", "scene0426_01", "scene0426_02", "scene0426_03", "scene0427_00", "scene0430_00", "scene0430_01", "scene0432_00", "scene0432_01", "scene0435_00", "scene0435_01", "scene0435_02", "scene0435_03", "scene0441_00", "scene0458_00", "scene0458_01", "scene0461_00", "scene0462_00", "scene0474_00", "scene0474_01", "scene0474_02", "scene0474_03", "scene0474_04", "scene0474_05", "scene0488_00", "scene0488_01", "scene0490_00", "scene0494_00", "scene0496_00", "scene0500_00", "scene0500_01", "scene0518_00", "scene0527_00", "scene0535_00", "scene0549_00", "scene0549_01", "scene0550_00", "scene0552_00", "scene0552_01", "scene0553_00", "scene0553_01", "scene0553_02", "scene0558_00", "scene0558_01", "scene0558_02", "scene0559_00", "scene0559_01", "scene0559_02", "scene0565_00", "scene0568_00", "scene0568_01", "scene0568_02", "scene0574_00", "scene0574_01", "scene0574_02", "scene0575_00", "scene0575_01", "scene0575_02", "scene0578_00", "scene0578_01", "scene0578_02", "scene0580_00", "scene0580_01", "scene0583_00", "scene0583_01", "scene0583_02", "scene0591_00", "scene0591_01", "scene0591_02", "scene0593_00", "scene0593_01", "scene0595_00", "scene0598_00", "scene0598_01", "scene0598_02", "scene0599_00", "scene0599_01", "scene0599_02", "scene0606_00", "scene0606_01", "scene0606_02", "scene0607_00", "scene0607_01", "scene0608_00", "scene0608_01", "scene0608_02", "scene0609_00", "scene0609_01", "scene0609_02", "scene0609_03", "scene0616_00", "scene0616_01", "scene0618_00", "scene0621_00", "scene0629_00", "scene0629_01", "scene0629_02", "scene0633_00", "scene0633_01", "scene0643_00", "scene0644_00", "scene0645_00", "scene0645_01", "scene0645_02", "scene0647_00", "scene0647_01", "scene0648_00", "scene0648_01", "scene0651_00", "scene0651_01", "scene0651_02", "scene0652_00", "scene0653_00", "scene0653_01", "scene0655_00", "scene0655_01", "scene0655_02", "scene0658_00", "scene0660_00", "scene0663_00", "scene0663_01", "scene0663_02", "scene0664_00", "scene0664_01", "scene0664_02", "scene0665_00", "scene0665_01", "scene0670_00", "scene0670_01", "scene0671_00", "scene0671_01", "scene0678_00", "scene0678_01", "scene0678_02", "scene0684_00", "scene0684_01", "scene0685_00", "scene0685_01", "scene0685_02", "scene0686_00", "scene0686_01", "scene0686_02", "scene0689_00", "scene0690_00", "scene0690_01", "scene0693_00", "scene0693_01", "scene0693_02", "scene0695_00", "scene0695_01", "scene0695_02", "scene0695_03", "scene0696_00", "scene0696_01", "scene0696_02", "scene0697_00", "scene0697_01", "scene0697_02", "scene0697_03", "scene0699_00", "scene0700_00", "scene0700_01", "scene0700_02", "scene0701_00", "scene0701_01", "scene0701_02", "scene0702_00", "scene0702_01", "scene0702_02", "scene0704_00", "scene0704_01"] + # print('There are %d scenes' % len(seq_name_list)) + + # Step 1: use Cropformer to get 2D instance masks for all sequences. + # parallel_compute(f'python third_party/detectron2/projects/CropFormer/demo_cropformer/mask_predict.py --config-file third_party/detectron2/projects/CropFormer/configs/entityv2/entity_segmentation/mask2former_hornet_3x.yaml --root {root} --image_path_pattern {image_path_pattern} --dataset {args.dataset} --seq_name_list %s --opts MODEL.WEIGHTS {cropformer_path}', 'predict mask', 'cuda', CUDA_LIST, seq_name_list) + t1 = time.time() + print('finish predict mask, cropformer time: ', t1 - t0, 'sec') + print('avg per scene time: ', (t1 - t0) / len(seq_name_list), 'sec') + # # Step 2: Mask clustering using our proposed method. + parallel_compute(f'python main.py --config {config} --seq_name_list %s', 'mask clustering', 'cuda', CUDA_LIST, seq_name_list) + t2 = time.time() + print('finish mask clustering, mask clustering time: ', t2 - t1) + print('avg per scene time: ', (t2 - t1) / len(seq_name_list)) + # Step 3: Evaluate the class-agnostic results. + # os.system(f'python -m evaluation.evaluate --pred_path data/prediction/{config}_class_agnostic --gt_path {gt} --dataset {dataset} --no_class') + + # Step 4: Get the open-vocabulary semantic features for each 2D masks. + parallel_compute(f'python -m semantics.get_open-voc_features --config {config} --seq_name_list %s', 'get open-vocabulary semantic features using CLIP', 'cuda', CUDA_LIST, seq_name_list) + + # Step 5: Get the text CLIP features for each label. + # get_label_text_feature(CUDA_LIST[0]) + + # Step 6: Get labels for each 3D instances. + parallel_compute(f'python -m semantics.open-voc_query --config {config}', 'get text labels', 'cpu', CUDA_LIST, seq_name_list) + t3 = time.time() + print('finish get text labels, get text labels time: ', t3 - t2) + print('avg per scene time: ', (t3 - t2) / len(seq_name_list)) + # Step 7: Evaluate the class-aware results. + # os.system(f'python -m evaluation.evaluate --pred_path data/prediction/{config} --gt_path {gt} --dataset {dataset}') + + print('total time', (time.time() - t0)//60, 'min') + print('Average time', (time.time() - t0) / len(seq_name_list), 'sec') + +if __name__ == '__main__': + args = get_args() + main(args) diff --git a/MaskClustering/run_itw.py b/MaskClustering/run_itw.py new file mode 100644 index 0000000000000000000000000000000000000000..301250bb3383d2c9625ac9af15573b9dfc91b09f --- /dev/null +++ b/MaskClustering/run_itw.py @@ -0,0 +1,88 @@ +import os +from tqdm import tqdm +import time +from utils.config import get_args +from pathlib import Path +import pickle +import torch +import open_clip +from open_clip import tokenizer +import numpy as np + +def load_clip(): + print(f'[INFO] loading CLIP model...') + model, _, _ = open_clip.create_model_and_transforms("ViT-H-14", pretrained="laion2b_s32b_b79k") + model.cuda() + model.eval() + print(f'[INFO]', ' finish loading CLIP model...') + return model + +def extract_text_feature(descriptions, clip_model, target_path): + text_tokens = tokenizer.tokenize(descriptions).cuda() + with torch.no_grad(): + text_features = clip_model.encode_text(text_tokens).float() + text_features /= text_features.norm(dim=-1, keepdim=True) + text_features = text_features.cpu().numpy() + + text_features_dict = {} + for i, description in enumerate(descriptions): + text_features_dict[description] = text_features[i] + + np.save(os.path.join(target_path, "text_features.npy"), text_features_dict) + return text_features_dict + + +clip_model = load_clip() + +def execute_commands(commands_list, command_type, process_num): + print('====> Start', command_type) + from multiprocessing import Pool + pool = Pool(process_num) + for _ in tqdm(pool.imap_unordered(os.system, commands_list), total=len(commands_list)): + pass + pool.close() + pool.join() + pool.terminate() + print('====> Finish', command_type) + + +def parallel_compute(general_command, command_name, resource_type, cuda_list, seq_name_list): + cuda_num = len(cuda_list) + + if resource_type == 'cuda': + commands = [] + for i, cuda_id in enumerate(cuda_list): + process_seq_name = seq_name_list[i::cuda_num] + if len(process_seq_name) == 0: + continue + process_seq_name = '+'.join(process_seq_name) + command = f'CUDA_VISIBLE_DEVICES={cuda_id} {general_command % process_seq_name}' + commands.append(command) + execute_commands(commands, command_name, cuda_num) + elif resource_type == 'cpu': + commands = [] + for seq_name in seq_name_list: + commands.append(f'{general_command} --seq_name {seq_name}') + execute_commands(commands, command_name, cuda_num) + + +def main(args): + CUDA_LIST = args.devices + config = "itw" + cropformer_path = args.cropformer_path + + root = f'data/{config}/processed' + image_path_pattern = 'color/*.jpg' # stride = 10 + # seq_name_list = get_seq_name_list(config) + seq_name_list = os.listdir(root) + + with open("labels.txt", "r") as f: + labels = f.read().split(";") + extract_text_feature(labels, clip_model, f"{root}/{seq_name_list[0]}") + + # Step 6: Get labels for each 3D instances. + parallel_compute(f'PYTHONPATH=. python semantics/itw-ovq.py --config {config}', 'get text labels', 'cpu', CUDA_LIST, seq_name_list) + +if __name__ == '__main__': + args = get_args() + main(args) diff --git a/MaskClustering/run_scannet.py b/MaskClustering/run_scannet.py new file mode 100644 index 0000000000000000000000000000000000000000..ede94925321f8742269de912b4cbafa6394d2d31 --- /dev/null +++ b/MaskClustering/run_scannet.py @@ -0,0 +1,167 @@ +import os +from tqdm import tqdm +import time +from utils.config import get_args +import argparse + +def execute_commands(commands_list, command_type, process_num): + print('====> Start', command_type) + from multiprocessing import Pool + pool = Pool(process_num) + for _ in tqdm(pool.imap_unordered(os.system, commands_list), total=len(commands_list)): + pass + pool.close() + pool.join() + pool.terminate() + print('====> Finish', command_type) + +def get_seq_name_list(config): + # Map config names to actual data directories + config_dir = config + if config in ['scannet18', 'scannet_dust3r_posed', 'scannet_dust3r_unposed', + 'scannet_dust3r_posed_15', 'scannet_dust3r_posed_25', 'scannet_dust3r_posed_35', + 'scannet_dust3r_unposed_15', 'scannet_dust3r_unposed_25', 'scannet_dust3r_unposed_35']: + if config == 'scannet18': + config_dir = 'scannet' + else: + config_dir = config + elif config in ['scannetpp_v2_dust3r_posed', 'scannetpp_v2_dust3r_unposed']: + config_dir = config + + root = f'data/{config_dir}/processed' + if not os.path.exists(root): + raise FileNotFoundError(f"Directory not found: {root}") + seq_name_list = os.listdir(root) + return seq_name_list + +def filter_completed_step1(config, seq_name_list): + """Filter scenes that already have CropFormer masks""" + config_dir = config if config != 'scannet18' else 'scannet' + root = f'data/{config_dir}/processed' + + filtered = [] + for seq_name in seq_name_list: + mask_dir = os.path.join(root, seq_name, 'output/mask') + # Check if mask directory exists and has mask files + if not os.path.exists(mask_dir) or len(os.listdir(mask_dir)) == 0: + filtered.append(seq_name) + + print(f'Step 1 (CropFormer): {len(filtered)}/{len(seq_name_list)} scenes need processing') + return filtered + +def filter_completed_step2(config, seq_name_list): + """Filter scenes that already have mask clustering results""" + config_dir = config if config != 'scannet18' else 'scannet' + root = f'data/{config_dir}/processed' + + filtered = [] + for seq_name in seq_name_list: + object_file = os.path.join(root, seq_name, 'output/object/object_dict.pkl') + if not os.path.exists(object_file): + filtered.append(seq_name) + + print(f'Step 2 (Mask Clustering): {len(filtered)}/{len(seq_name_list)} scenes need processing') + return filtered + +def filter_completed_step3(config, seq_name_list): + """Filter scenes that already have CLIP features""" + config_dir = config if config != 'scannet18' else 'scannet' + root = f'data/{config_dir}/processed' + + filtered = [] + for seq_name in seq_name_list: + features_file = os.path.join(root, seq_name, 'output/features_clip.npy') + if not os.path.exists(features_file): + filtered.append(seq_name) + + print(f'Step 3 (CLIP Features): {len(filtered)}/{len(seq_name_list)} scenes need processing') + return filtered + +def parallel_compute(general_command, command_name, resource_type, cuda_list, seq_name_list): + cuda_num = len(cuda_list) + + if resource_type == 'cuda': + commands = [] + for i, cuda_id in enumerate(cuda_list): + process_seq_name = seq_name_list[i::cuda_num] + if len(process_seq_name) == 0: + continue + process_seq_name = '+'.join(process_seq_name) + command = f'CUDA_VISIBLE_DEVICES={cuda_id} {general_command % process_seq_name}' + commands.append(command) + execute_commands(commands, command_name, cuda_num) + elif resource_type == 'cpu': + commands = [] + for seq_name in seq_name_list: + commands.append(f'{general_command} --seq_name {seq_name}') + execute_commands(commands, command_name, cuda_num) + +def main(args): + config = args.config + cuda_list = args.cuda_list + cropformer_path = args.cropformer_path + dataset = args.dataset if args.dataset else config + + # Map config names to actual data directories + config_dir = config + if config == 'scannet18': + config_dir = 'scannet' + + root = f'data/{config_dir}/processed' + image_path_pattern = 'color/*.jpg' + + t0 = time.time() + + # Check if the processed directory exists + if not os.path.exists(root): + print(f'Processed directory not found: {root}') + print('Please run export scripts first!') + return + + seq_name_list = get_seq_name_list(config) + print(f'There are {len(seq_name_list)} scenes exported and ready to process in {config}') + + # Step 1: use Cropformer to get 2D instance masks for all sequences. + seq_list_step1 = filter_completed_step1(config, seq_name_list) + if len(seq_list_step1) > 0: + parallel_compute(f'python third_party/detectron2/projects/CropFormer/demo_cropformer/mask_predict.py --config-file third_party/detectron2/projects/CropFormer/configs/entityv2/entity_segmentation/mask2former_hornet_3x.yaml --root {root} --image_path_pattern {image_path_pattern} --dataset {dataset} --seq_name_list %s --opts MODEL.WEIGHTS {cropformer_path}', 'predict mask', 'cuda', cuda_list, seq_list_step1) + else: + print('Step 1: All scenes already have CropFormer masks, skipping...') + + # Step 2: Mask clustering using our proposed method. + seq_list_step2 = filter_completed_step2(config, seq_name_list) + if len(seq_list_step2) > 0: + parallel_compute(f'python main.py --config {config} --seq_name_list %s', 'mask clustering', 'cuda', cuda_list, seq_list_step2) + else: + print('Step 2: All scenes already have mask clustering results, skipping...') + + # Step 3: Get the open-vocabulary semantic features for each 2D masks. + seq_list_step3 = filter_completed_step3(config, seq_name_list) + if len(seq_list_step3) > 0: + parallel_compute(f'python -m semantics.get_open-voc_features --config {config} --dataset {config} --seq_name_list %s', 'get open-vocabulary semantic features using CLIP', 'cuda', cuda_list, seq_list_step3) + else: + print('Step 3: All scenes already have CLIP features, skipping...') + + # Step 4: Get labels for each 3D instances. + parallel_compute(f'python -m semantics.open-voc_query --config {config} --dataset {config}', 'get text labels', 'cpu', cuda_list, seq_name_list) + + print('total time', (time.time() - t0)//60, 'min') + print('Average time', (time.time() - t0) / len(seq_name_list), 'sec') + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Run mask clustering on ScanNet datasets') + parser.add_argument('--config', type=str, required=True, + choices=['scannet18', 'scannet_dust3r_posed_15', 'scannet_dust3r_posed_25', 'scannet_dust3r_posed_35', + 'scannet_dust3r_unposed_15', 'scannet_dust3r_unposed_25', 'scannet_dust3r_unposed_35', + 'scannetpp_v2_dust3r_posed', 'scannetpp_v2_dust3r_unposed'], + help='Config name for the dataset') + parser.add_argument('--dataset', type=str, default=None, + help='Dataset name to pass to CropFormer (defaults to config name)') + parser.add_argument('--cuda_list', type=int, nargs='+', default=[0, 1, 2, 3], + help='List of CUDA device IDs to use (e.g., --cuda_list 0 1 2 3)') + parser.add_argument('--cropformer_path', type=str, default='Mask2Former_hornet_3x_576d0b.pth', + help='Path to CropFormer model weights') + + args = parser.parse_args() + main(args) + diff --git a/MaskClustering/semantics/3dtrack_open-voc_query.py b/MaskClustering/semantics/3dtrack_open-voc_query.py new file mode 100644 index 0000000000000000000000000000000000000000..c40be18a256c171c985e3ed383b8160b1f893d3e --- /dev/null +++ b/MaskClustering/semantics/3dtrack_open-voc_query.py @@ -0,0 +1,207 @@ +''' + This script is used to generate the semantic labels for the objects in the scene. +''' +from utils.config import get_dataset, get_args +from dataset.scannet import ScanNetDataset +import os +import numpy as np +import open_clip +import cv2 +from PIL import Image +import torch +from torch.utils.data import TensorDataset, DataLoader +from tqdm import tqdm +from utils.config import update_args +import argparse + + +LEVELS = 3 + +def load_clip(device): + print(f'[INFO] loading CLIP model...') + model, _, preprocess = open_clip.create_model_and_transforms("ViT-H-14", pretrained="laion2b_s32b_b79k") + model.to(device) + model.eval() + print(f'[INFO]', ' finish loading CLIP model...') + return model, preprocess + +def box_multi_level(bbox, shape, level, expansion_ratio): + left, top, right, bottom = bbox + + + if level == 0: + return left, top, right , bottom + + x_exp = int(abs(right - left)*expansion_ratio * level) + y_exp = int(abs(bottom - top)*expansion_ratio * level) + return max(0, left - x_exp), max(0, top - y_exp), min(shape[1], right + x_exp), min(shape[0], bottom + y_exp) + +def get_cropped_images(object, scene_id, preprocess, preloaded_images, num_images=5, expansion_ratio=0.1): + croped_images = [] + images = [] + for frame in object['frames'][:num_images]: + image = preloaded_images[frame['frame_id']] + + x1, y1, x2, y2 = frame['bbox'] + x1, y1, x2, y2 = np.round([x1, y1, x2, y2]).astype(int) + for level in range(LEVELS): + x1_, y1_, x2_, y2_ = box_multi_level((x1, y1, x2, y2), np.asarray(image).shape, level, expansion_ratio) + if x1_ == x2_ or y1_ == y2_: + continue + pil_image = pad_into_square(Image.fromarray(cv2.cvtColor(image[y1_:y2_, x1_:x2_], cv2.COLOR_BGR2RGB))) + croped_images.append(preprocess(pil_image)) + images.append(np.asarray(pil_image.resize((64, 64)))) + if len(croped_images) == 0: + return None, None + return torch.stack(croped_images), np.concatenate(images, axis=1)[..., ::-1] + +def pad_into_square(image): + width, height = image.size + new_size = max(width, height) + new_image = Image.new("RGB", (new_size, new_size), (255,255,255)) + left = (new_size - width) // 2 + top = (new_size - height) // 2 + new_image.paste(image, (left, top)) + return new_image + +def preload_images(scene, frame_ids): + preloaded_images = {} + for frame_id in frame_ids: + img_path = f'/home/jovyan/users/lemeshko/mmdetection3d/data/scannet/posed_images/{scene}/{str(frame_id).zfill(5)}.jpg' + image = cv2.imread(img_path) + preloaded_images[frame_id] = image + return preloaded_images + +def custom_probs(feature, label_text_features): + object_feature = feature #np.mean(feature, axis=0, keepdims=True) + print(object_feature.shape, np.mean(feature, axis=0, keepdims=True).shape) + + raw_similarity = object_feature @ label_text_features.T + raw_similarity = np.sum(raw_similarity, axis=0, keepdims=True) + + exp_sim = np.exp(raw_similarity) + prob = exp_sim / np.sum(exp_sim, axis=1, keepdims=True) + probs = np.max(prob, axis=0) + max_label_id = np.argmax(probs) + prob = probs[max_label_id] + return prob, max_label_id + +def main(args): + scenes = np.loadtxt('/home/jovyan/users/bulat/workspace/3drec/Indoor/MaskClustering/splits/scannet200_subset.txt', dtype=str) + # scenes = scenes[:1] + # scenes = ["scene0011_00"] + + for scene in tqdm(scenes): + + frame_ids = set() + + args.seq_name = scene + + pred_dir = os.path.join('data/prediction', args.config, args.exp_name) + + if os.path.exists(f'{pred_dir}/{args.seq_name}.npz'): + continue + dataset = ScanNetDataset(scene, use_templates=True) + total_point_num = dataset.get_scene_points().shape[0] + + label_features_dict = dataset.get_label_features() + label_text_features = np.stack(list(label_features_dict.values())) + descriptions = list(label_features_dict.keys()) + + scene_name = dataset.seq_name + logs_path = f'logs/{args.exp_name}/{scene}' + + # Загружаем данные в новом формате (список объектов) + object_list = np.load(f"/home/jovyan/users/bulat/workspace/3drec/det/OV/mask_proj/outputs/30/{scene}/mask_data.npy", allow_pickle=True) + # print(f'[INFO] loaded {object_list} objects') + + label2id = dataset.get_label_id()[0] + + os.makedirs(pred_dir, exist_ok=True) + num_instance = len(object_list) + pred_dict = { + "pred_masks": np.zeros((total_point_num, num_instance), dtype=bool), + "pred_score": np.ones(num_instance), + "pred_classes" : np.zeros(num_instance, dtype=np.int32) + } + + # Собираем все frame_ids из всех объектов + for object_data in object_list: + frame_ids.update([frame['frame_id'] for frame in object_data['frames']]) + frame_ids = list(frame_ids) + + preloaded_images = preload_images(scene, frame_ids) + + + + # Исправляем итерацию по объектам + for idx, object_data in enumerate(object_list): + croped_images, saved_images = get_cropped_images(object_data, scene_name, args.preprocess, preloaded_images=preloaded_images, num_images=args.num_images) + if croped_images is None: + print(f'[INFO] no croped images for object {idx}') + continue + + bs = 32 + chunks = torch.chunk(croped_images, max(1, len(croped_images) // bs)) + + features = [] + for images in chunks: + # images = images[0] + images = images.to(args.device) + with torch.no_grad(): + image_features = args.model.encode_image(images).float() + image_features /= image_features.norm(dim=-1, keepdim=True) + image_features = image_features.cpu().numpy() + for f in image_features: + features.append(f) + + feature = np.stack(features) + object_feature = np.mean(feature, axis=0, keepdims=True) + + raw_similarity = np.dot(object_feature, label_text_features.T) + exp_sim = np.exp(raw_similarity * 100) + prob = exp_sim / np.sum(exp_sim, axis=1, keepdims=True) + probs = np.max(prob, axis=0) + max_label_id = np.argmax(probs) + prob = probs[max_label_id] + pred_dict['pred_score'][idx] = prob + + label_id = label2id[descriptions[max_label_id]] + pred_dict['pred_classes'][idx] = label_id + + point_ids = object_data['mask'] + binary_mask = np.zeros(total_point_num, dtype=bool) + binary_mask[list(point_ids)] = True + pred_dict['pred_masks'][:, idx] = binary_mask + if args.debug: + os.makedirs(logs_path, exist_ok=True) + cv2.imwrite(f'{logs_path}/{str(idx).zfill(5)}_{dataset.get_label_id()[1][label_id]}({prob:.2f}).jpg', saved_images) + print(idx, label_id, dataset.get_label_id()[1][label_id], "confidence:", prob) + pred_classes = pred_dict['pred_classes'] + # pred_classes = [dataset.get_label_id()[1][i] for i in pred_classes] + + pred_dict['pred_masks'] = pred_dict['pred_masks'][:, pred_classes != 0] + pred_dict['pred_score'] = pred_dict['pred_score'][pred_classes != 0] + pred_dict['pred_classes'] = pred_dict['pred_classes'][pred_classes != 0] + + + + np.savez(f'{pred_dir}/{args.seq_name}.npz', **pred_dict) + +if __name__ == '__main__': + + parser = argparse.ArgumentParser() + parser.add_argument('--config', type=str, default='scannet') + parser.add_argument('--debug', action="store_true") + parser.add_argument('--exp_name', type=str, default='baseline') + parser.add_argument('--device', type=str, default='cuda:0') + parser.add_argument('--num_images', type=int, default=5) + parser.add_argument('--num_workers', '-n', type=int, default=1) + parser.add_argument('--path_to_predictions', type=str, default='/home/jovyan/users/bulat/workspace/3drec/Indoor/Grounded-SAM-2/results/gsam_result/scannet200/yolo/memory_150_classes_198') + parser.add_argument('--job_id', '-i', type=int, default=0) + args = parser.parse_args() + args = update_args(args) + model, preprocess = load_clip(args.device) + args.model = model + args.preprocess = preprocess + main(args) diff --git a/MaskClustering/semantics/c_open-voc_query.py b/MaskClustering/semantics/c_open-voc_query.py new file mode 100644 index 0000000000000000000000000000000000000000..11cbcbeacc769401598a3fb6e7a9893cbd7e03f4 --- /dev/null +++ b/MaskClustering/semantics/c_open-voc_query.py @@ -0,0 +1,196 @@ +''' + This script is used to generate the semantic labels for the objects in the scene. +''' +from utils.config import get_dataset, get_args +from dataset.scannet import ScanNetDataset +import os +import numpy as np +import open_clip +import cv2 +from PIL import Image +import torch +from torch.utils.data import TensorDataset, DataLoader +from tqdm import tqdm +from utils.config import update_args +import argparse + + +LEVELS = 3 + +def load_clip(device): + print(f'[INFO] loading CLIP model...') + model, _, preprocess = open_clip.create_model_and_transforms("ViT-H-14", pretrained="laion2b_s32b_b79k") + model.to(device) + model.eval() + print(f'[INFO]', ' finish loading CLIP model...') + return model, preprocess + +def box_multi_level(bbox, shape, level, expansion_ratio): + left, top, right, bottom = bbox + + + if level == 0: + return left, top, right , bottom + + x_exp = int(abs(right - left)*expansion_ratio) * level + y_exp = int(abs(bottom - top)*expansion_ratio) * level + return max(0, left - x_exp), max(0, top - y_exp), min(shape[1], right + x_exp), min(shape[0], bottom + y_exp) + +def get_cropped_images(object, scene_id, preprocess, num_images=5, expansion_ratio=0.1, preloaded_images=None): + croped_images = [] + images = [] + for frame in object['frames'][:num_images]: + image = preloaded_images[frame['frame_id']] + + x1, y1, x2, y2 = frame['bbox'] + for level in range(LEVELS): + x1, y1, x2, y2 = box_multi_level((x1, y1, x2, y2), np.asarray(image).shape, level, expansion_ratio) + pil_image = pad_into_square(Image.fromarray(cv2.cvtColor(image[y1:y2, x1:x2], cv2.COLOR_BGR2RGB))) + croped_images.append(preprocess(pil_image)) + images.append(np.asarray(pil_image.resize((64, 64)))) + return torch.stack(croped_images), np.concatenate(images, axis=1)[..., ::-1] + +def pad_into_square(image): + width, height = image.size + new_size = max(width, height) + new_image = Image.new("RGB", (new_size, new_size), (255,255,255)) + left = (new_size - width) // 2 + top = (new_size - height) // 2 + new_image.paste(image, (left, top)) + return new_image + +def preload_images(scene, frame_ids): + preloaded_images = {} + for frame_id in frame_ids: + img_path = f'/home/jovyan/users/lemeshko/mmdetection3d/data/scannet/posed_images/{scene}/{str(frame_id).zfill(5)}.jpg' + image = cv2.imread(img_path) + preloaded_images[frame_id] = image + return preloaded_images + +def custom_probs(feature, label_text_features): + object_feature = feature #np.mean(feature, axis=0, keepdims=True) + print(object_feature.shape, np.mean(feature, axis=0, keepdims=True).shape) + + raw_similarity = object_feature @ label_text_features.T + raw_similarity = np.sum(raw_similarity, axis=0, keepdims=True) + + exp_sim = np.exp(raw_similarity) + prob = exp_sim / np.sum(exp_sim, axis=1, keepdims=True) + probs = np.max(prob, axis=0) + max_label_id = np.argmax(probs) + prob = probs[max_label_id] + return prob, max_label_id + +def main(args): + scenes = np.loadtxt('/home/jovyan/users/bulat/workspace/3drec/Indoor/MaskClustering/splits/scannet200_subset.txt', dtype=str) + scenes = scenes[args.job_id::args.num_workers] + # scenes = scenes[:1] + + for scene in tqdm(scenes): + + frame_ids = set() + + args.seq_name = scene + + pred_dir = os.path.join('data/prediction', args.config, args.exp_name) + + if os.path.exists(f'{pred_dir}/{args.seq_name}.npz'): + continue + dataset = ScanNetDataset(scene) + total_point_num = dataset.get_scene_points().shape[0] + + label_features_dict = dataset.get_label_features() + label_text_features = np.stack(list(label_features_dict.values())) + descriptions = list(label_features_dict.keys()) + + scene_name = dataset.seq_name + logs_path = f'logs/{args.exp_name}/{scene}' + object_dict = np.load(os.path.join(args.path_to_predictions, scene_name, 'infos.npy'), allow_pickle=True).item() + + label2id = dataset.get_label_id()[0] + print(label2id) + print(dataset.get_label_id()[1]) + + os.makedirs(pred_dir, exist_ok=True) + num_instance = len(object_dict) + pred_dict = { + "pred_masks": np.zeros((total_point_num, num_instance), dtype=bool), + "pred_score": np.ones(num_instance), + "pred_classes" : np.zeros(num_instance, dtype=np.int32) + } + + for key, object in object_dict.items(): + frame_ids.update([a['frame_id'] for a in object['frames']]) + frame_ids = list(frame_ids) + + preloaded_images = preload_images(scene, frame_ids) + print(scene_name) + + + for idx, (key, object) in enumerate(object_dict.items()): + croped_images, saved_images = get_cropped_images(object, scene_name, args.preprocess, args.num_images, preloaded_images=preloaded_images) + + + bs = 32 + chunks = torch.chunk(croped_images, max(1, len(croped_images) // bs)) + + features = [] + for images in chunks: + # images = images[0] + images = images.to(args.device) + with torch.no_grad(): + image_features = args.model.encode_image(images).float() + image_features /= image_features.norm(dim=-1, keepdim=True) + image_features = image_features.cpu().numpy() + for f in image_features: + features.append(f) + + feature = np.stack(features) + object_feature = np.mean(feature, axis=0, keepdims=True) + + raw_similarity = np.dot(object_feature, label_text_features.T) + exp_sim = np.exp(raw_similarity * 100) + prob = exp_sim / np.sum(exp_sim, axis=1, keepdims=True) + probs = np.max(prob, axis=0) + max_label_id = np.argmax(probs) + prob = probs[max_label_id] + pred_dict['pred_score'][idx] = prob + + label_id = label2id[descriptions[max_label_id]] + pred_dict['pred_classes'][idx] = label_id + + point_ids = object['mask'] + binary_mask = np.zeros(total_point_num, dtype=bool) + binary_mask[list(point_ids)] = True + pred_dict['pred_masks'][:, idx] = binary_mask + if args.debug: + os.makedirs(logs_path, exist_ok=True) + cv2.imwrite(f'{logs_path}/{str(key).zfill(5)}_{dataset.get_label_id()[1][label_id]}({prob:.2f}).jpg', saved_images) + print(key, label_id, dataset.get_label_id()[1][label_id], "confidence:", prob) + pred_classes = pred_dict['pred_classes'] + pred_classes = [dataset.get_label_id()[1][i] for i in pred_classes] + # remove classes with label == 0 + + + + + + np.savez(f'{pred_dir}/{args.seq_name}.npz', **pred_dict) + +if __name__ == '__main__': + + parser = argparse.ArgumentParser() + parser.add_argument('--config', type=str, default='scannet') + parser.add_argument('--debug', action="store_true") + parser.add_argument('--exp_name', type=str, default='baseline') + parser.add_argument('--device', type=str, default='cuda:0') + parser.add_argument('--num_images', type=int, default=5) + parser.add_argument('--num_workers', '-n', type=int, default=1) + parser.add_argument('--path_to_predictions', type=str, default='/home/jovyan/users/bulat/workspace/3drec/Indoor/Grounded-SAM-2/results/gsam_result/scannet200/yolo/memory_150_classes_198') + parser.add_argument('--job_id', '-i', type=int, default=0) + args = parser.parse_args() + args = update_args(args) + model, preprocess = load_clip(args.device) + args.model = model + args.preprocess = preprocess + main(args) diff --git a/MaskClustering/semantics/c_open-voc_query_optimized.py b/MaskClustering/semantics/c_open-voc_query_optimized.py new file mode 100644 index 0000000000000000000000000000000000000000..2171763023b5f3478ee2bcbd7cc52baaac927362 --- /dev/null +++ b/MaskClustering/semantics/c_open-voc_query_optimized.py @@ -0,0 +1,353 @@ +''' + Оптимизированная версия скрипта для генерации семантических меток объектов в сцене. + Оптимизации: + - Многопоточная обработка I/O операций + - Кэширование загруженных изображений + - Пакетная обработка + - Оптимизированные операции с NumPy +''' +from utils.config import get_dataset, get_args +from dataset.scannet import ScanNetDataset +import os +import numpy as np +import open_clip +import cv2 +from PIL import Image +import torch +from torch.utils.data import TensorDataset, DataLoader +from tqdm import tqdm +from utils.config import update_args +import argparse +from datetime import datetime +import shutil +from concurrent.futures import ThreadPoolExecutor +from functools import lru_cache +from typing import Dict, List, Tuple + +LEVELS = 3 + +# Глобальный кэш для изображений +CACHE_SIZE = 1000 + +def load_clip(device): + print(f'[INFO] loading CLIP model...') + model, _, preprocess = open_clip.create_model_and_transforms("ViT-H-14", pretrained="laion2b_s32b_b79k") + model.to(device) + model.eval() + print(f'[INFO] finish loading CLIP model...') + return model, preprocess + +def box_multi_level(bbox, shape, level, expansion_ratio): + left, top, right, bottom = bbox + if level == 0: + return left, top, right, bottom + + x_exp = int(abs(right - left) * expansion_ratio) * level + y_exp = int(abs(bottom - top) * expansion_ratio) * level + return max(0, left - x_exp), max(0, top - y_exp), min(shape[1], right + x_exp), min(shape[0], bottom + y_exp) + +@lru_cache(maxsize=CACHE_SIZE) +def load_image_cached(img_path: str) -> np.ndarray: + """Кэшированная загрузка изображений""" + if not os.path.exists(img_path): + return None + return cv2.imread(img_path) + +@lru_cache(maxsize=CACHE_SIZE) +def load_mask_cached(mask_path: str) -> np.ndarray: + """Кэшированная загрузка масок""" + if not os.path.exists(mask_path): + return None + return np.load(mask_path, allow_pickle=True) + +def process_single_frame(args_tuple): + """Обработка одного кадра - функция для многопоточности I/O""" + frame, key, scene_id, img_base_path, mask_base_path, expansion_ratio = args_tuple + + img_path = f'{img_base_path}/{str(frame["frame_id"]).zfill(5)}.jpg' + mask_path = os.path.join(mask_base_path, frame['mask_path']) + + try: + image = load_image_cached(img_path) + mask = load_mask_cached(mask_path) + + if image is None or mask is None: + return None + + # Применяем маску более эффективно + mask_indices = mask == key + if not np.any(mask_indices): + return None + + image = image.copy() # Создаем копию только когда нужно + image[mask_indices] = (image[mask_indices] * 0.8).astype(np.uint8) + + x1, y1, x2, y2 = frame['bbox'] + cropped_images = [] + + for level in range(LEVELS): + x1, y1, x2, y2 = box_multi_level((x1, y1, x2, y2), image.shape, level, expansion_ratio) + + # Обрезаем изображение + cropped = image[y1:y2, x1:x2] + if cropped.size == 0: + continue + + # Конвертируем BGR -> RGB один раз + rgb_image = cv2.cvtColor(cropped, cv2.COLOR_BGR2RGB) + pil_image = pad_into_square(Image.fromarray(rgb_image)) + + cropped_images.append(np.array(pil_image)) + + return cropped_images + + except Exception as e: + print(f"Ошибка обработки кадра {frame['frame_id']}: {e}") + return None + +def get_cropped_images_parallel(key, object_data, scene_id, preprocess, num_images=5, expansion_ratio=0.1, num_workers=4): + """Параллельная обработка изображений с использованием ThreadPoolExecutor""" + img_base_path = f'/home/jovyan/users/lemeshko/mmdetection3d/data/scannet/posed_images/{scene_id}' + mask_base_path = f'/home/jovyan/users/lemeshko/scripts/gsam_result/scannet200/yolo/{scene_id}' + + frames = object_data['frames'][:num_images] + + # Подготавливаем аргументы для параллельной обработки + args_list = [ + (frame, key, scene_id, img_base_path, mask_base_path, expansion_ratio) + for frame in frames + ] + + all_cropped_images = [] + all_display_images = [] + + # Используем ThreadPoolExecutor для I/O операций (безопасно для CUDA) + with ThreadPoolExecutor(max_workers=num_workers) as executor: + results = list(executor.map(process_single_frame, args_list)) + + # Обрабатываем результаты + for result in results: + if result is not None: + for img_array in result: + # Применяем preprocess к PIL изображению + pil_img = Image.fromarray(img_array) + processed_img = preprocess(pil_img) + all_cropped_images.append(processed_img) + + # Для отображения + display_img = np.array(pil_img.resize((64, 64))) + all_display_images.append(display_img) + + if not all_cropped_images: + return torch.empty(0), np.array([]) + + return torch.stack(all_cropped_images), np.concatenate(all_display_images, axis=1)[..., ::-1] + +def pad_into_square(image): + """Оптимизированная функция для создания квадратного изображения""" + width, height = image.size + if width == height: + return image + + new_size = max(width, height) + new_image = Image.new("RGB", (new_size, new_size), (255, 255, 255)) + left = (new_size - width) // 2 + top = (new_size - height) // 2 + new_image.paste(image, (left, top)) + return new_image + +def batch_encode_images(model, images, device, batch_size=32): + """Пакетная обработка изображений через CLIP""" + if len(images) == 0: + return np.array([]) + + features = [] + for i in range(0, len(images), batch_size): + batch = images[i:i + batch_size].to(device) + with torch.no_grad(): + batch_features = model.encode_image(batch).float() + batch_features /= batch_features.norm(dim=-1, keepdim=True) + features.append(batch_features.cpu().numpy()) + + return np.vstack(features) if features else np.array([]) + +def process_objects_batch(object_items, args, scene_name, label_text_features, descriptions, label2id, total_point_num, logs_path, dataset): + """Обработка пакета объектов для улучшения эффективности""" + batch_results = [] + + for idx, (key, object_data) in object_items: + try: + cropped_images, saved_images = get_cropped_images_parallel( + key, object_data, scene_name, args.preprocess, args.num_images, + num_workers=args.image_workers + ) + + if len(cropped_images) == 0: + batch_results.append(None) + continue + + # Пакетная обработка через CLIP + features = batch_encode_images(args.model, cropped_images, args.device, batch_size=32) + + if features.size == 0: + batch_results.append(None) + continue + + # Вычисляем среднее по признакам + object_feature = np.mean(features, axis=0, keepdims=True) + + # Вычисляем схожесть + raw_similarity = np.dot(object_feature, label_text_features.T) + exp_sim = np.exp(raw_similarity * 100) + prob = exp_sim / np.sum(exp_sim, axis=1, keepdims=True) + probs = np.max(prob, axis=0) + max_label_id = np.argmax(probs) + prob_score = probs[max_label_id] + + label_id = label2id[descriptions[max_label_id]] + + # Создаем бинарную маску + point_ids = object_data['mask'] + binary_mask = np.zeros(total_point_num, dtype=bool) + binary_mask[list(point_ids)] = True + + result = { + 'idx': idx, + 'key': key, + 'binary_mask': binary_mask, + 'prob_score': prob_score, + 'label_id': label_id, + 'saved_images': saved_images if args.debug else None + } + + batch_results.append(result) + + if args.debug and saved_images is not None: + os.makedirs(logs_path, exist_ok=True) + cv2.imwrite(f'{logs_path}/{str(key).zfill(5)}_{dataset.get_label_id()[1][label_id]}({prob_score:.2f}).jpg', saved_images) + + except Exception as e: + print(f"Ошибка обработки объекта {key}: {e}") + batch_results.append(None) + + return batch_results + +def main(args): + scenes = np.loadtxt('splits/scannet200_subset.txt', dtype=str) + scenes = scenes[args.job_id::args.num_workers] + pred_dir = os.path.join('data/prediction', args.config, args.exp_name) + os.makedirs(pred_dir, exist_ok=True) + shutil.copy("semantics/c_open-voc_query_optimized.py", os.path.join(pred_dir, "c_open-voc_query_optimized.py")) + + for scene in tqdm(scenes): + args.seq_name = scene + + if os.path.exists(f'{pred_dir}/{args.seq_name}.npz'): + continue + + dataset = ScanNetDataset(scene) + total_point_num = dataset.get_scene_points().shape[0] + + label_features_dict = dataset.get_label_features() + label_text_features = np.stack(list(label_features_dict.values())) + descriptions = list(label_features_dict.keys()) + + scene_name = dataset.seq_name + logs_path = os.path.join('logs', args.exp_name, scene) + object_dict = np.load(f'/home/jovyan/users/lemeshko/scripts/gsam_result/scannet200/yolo/{scene_name}/infos.npy', allow_pickle=True).item() + + label2id = dataset.get_label_id()[0] + num_instance = len(object_dict) + + pred_dict = { + "pred_masks": np.zeros((total_point_num, num_instance), dtype=bool), + "pred_score": np.ones(num_instance), + "pred_classes": np.zeros(num_instance, dtype=np.int32) + } + + print(f"Обработка сцены {scene_name} с {num_instance} объектами") + + # Пакетная обработка объектов для лучшей эффективности + if args.batch_processing and num_instance > args.batch_size: + object_items = list(enumerate(object_dict.items())) + + # Разбиваем на пакеты + for batch_start in range(0, len(object_items), args.batch_size): + batch_end = min(batch_start + args.batch_size, len(object_items)) + batch = object_items[batch_start:batch_end] + + print(f"Обработка пакета {batch_start//args.batch_size + 1}/{(len(object_items) + args.batch_size - 1)//args.batch_size}") + + batch_results = process_objects_batch( + batch, args, scene_name, label_text_features, descriptions, + label2id, total_point_num, logs_path, dataset + ) + + # Сохраняем результаты пакета + for result in batch_results: + if result is not None: + idx = result['idx'] + pred_dict['pred_masks'][:, idx] = result['binary_mask'] + pred_dict['pred_score'][idx] = result['prob_score'] + pred_dict['pred_classes'][idx] = result['label_id'] + else: + # Последовательная обработка с оптимизированными функциями + for idx, (key, object_data) in enumerate(object_dict.items()): + cropped_images, saved_images = get_cropped_images_parallel( + key, object_data, scene_name, args.preprocess, args.num_images, + num_workers=args.image_workers + ) + + if len(cropped_images) == 0: + continue + + features = batch_encode_images(args.model, cropped_images, args.device, batch_size=32) + object_feature = np.mean(features, axis=0, keepdims=True) + + raw_similarity = np.dot(object_feature, label_text_features.T) + exp_sim = np.exp(raw_similarity * 100) + prob = exp_sim / np.sum(exp_sim, axis=1, keepdims=True) + probs = np.max(prob, axis=0) + max_label_id = np.argmax(probs) + prob_score = probs[max_label_id] + + pred_dict['pred_score'][idx] = prob_score + label_id = label2id[descriptions[max_label_id]] + pred_dict['pred_classes'][idx] = label_id + + point_ids = object_data['mask'] + binary_mask = np.zeros(total_point_num, dtype=bool) + binary_mask[list(point_ids)] = True + pred_dict['pred_masks'][:, idx] = binary_mask + + if args.debug: + os.makedirs(logs_path, exist_ok=True) + cv2.imwrite(f'{logs_path}/{str(key).zfill(5)}_{dataset.get_label_id()[1][label_id]}({prob_score:.2f}).jpg', saved_images) + + np.savez(f'{pred_dir}/{args.seq_name}.npz', **pred_dict) + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--config', type=str, default='scannet') + parser.add_argument('--debug', action="store_true") + parser.add_argument('--exp_name', type=str, default='baseline') + parser.add_argument('--device', type=str, default='cuda:0') + parser.add_argument('--num_images', type=int, default=5) + parser.add_argument('--num_workers', '-n', type=int, default=1) + parser.add_argument('--job_id', '-i', type=int, default=0) + # Параметры оптимизации (безопасные для CUDA) + parser.add_argument('--batch_processing', action="store_true", help="Пакетная обработка объектов") + parser.add_argument('--batch_size', type=int, default=10, help="Размер пакета объектов") + parser.add_argument('--image_workers', type=int, default=4, help="Количество потоков для загрузки изображений") + + args = parser.parse_args() + args = update_args(args) + + model, preprocess = load_clip(args.device) + args.model = model + args.preprocess = preprocess + + date = datetime.now().strftime("%Y-%m-%d_%H-%M-%S") + args.exp_name = f'{args.exp_name}' + + main(args) \ No newline at end of file diff --git a/MaskClustering/semantics/extract_label_featrues.py b/MaskClustering/semantics/extract_label_featrues.py new file mode 100644 index 0000000000000000000000000000000000000000..3e70cfa1e1e930e60e56934619dc13a0bff570f8 --- /dev/null +++ b/MaskClustering/semantics/extract_label_featrues.py @@ -0,0 +1,40 @@ +import open_clip +from open_clip import tokenizer +import torch +import numpy as np +from evaluation.constants import MATTERPORT_LABELS, SCANNET_LABELS, SCANNETPP_LABELS, SCANNET18_LABELS, SCANNETPP84_LABELS, SCANNETPP84_IDS, ARKIT_LABELS, ARKIT_IDS + +def load_clip(): + print(f'[INFO] loading CLIP model...') + model, _, _ = open_clip.create_model_and_transforms("ViT-H-14", pretrained="laion2b_s32b_b79k") + model.cuda() + model.eval() + print(f'[INFO]', ' finish loading CLIP model...') + return model + +def extract_text_feature(save_path, descriptions): + text_tokens = tokenizer.tokenize(descriptions).cuda() + with torch.no_grad(): + text_features = model.encode_text(text_tokens).float() + text_features /= text_features.norm(dim=-1, keepdim=True) + text_features = text_features.cpu().numpy() + + text_features_dict = {} + for i, description in enumerate(descriptions): + text_features_dict[description] = text_features[i] + + np.save(save_path, text_features_dict) + +def get_text_feature(text): + text_tokens = tokenizer.tokenize(text).cuda() + with torch.no_grad(): + text_features = model.encode_text(text_tokens).float() + return text_features.cpu().numpy() + +model = load_clip() +extract_text_feature('data/text_features/scannet.npy', SCANNET_LABELS) +extract_text_feature('data/text_features/scannetpp.npy', SCANNETPP_LABELS) +extract_text_feature('data/text_features/matterport3d.npy', MATTERPORT_LABELS) +extract_text_feature('data/text_features/scannet18.npy', SCANNET18_LABELS) +extract_text_feature('data/text_features/scannetpp84.npy', SCANNETPP84_LABELS) +extract_text_feature('data/text_features/arkit.npy', ARKIT_LABELS) \ No newline at end of file diff --git a/MaskClustering/semantics/get_open-voc_features.py b/MaskClustering/semantics/get_open-voc_features.py new file mode 100644 index 0000000000000000000000000000000000000000..c4b3da47a2ae96afc1739649171c713889435a63 --- /dev/null +++ b/MaskClustering/semantics/get_open-voc_features.py @@ -0,0 +1,153 @@ +''' + This script extracts open-vocabulary visual features for each mask following OpenMask3D. + For each mask, we crop the image with CROP_SCALES=3 scales based on the mask. + Then we extract the visual features using CLIP model and average these features as the mask feature. +''' + +import open_clip +import os +from PIL import Image +import numpy as np +import torch +from utils.config import get_args, get_dataset +from torch.utils.data import Dataset +from torch.utils.data import DataLoader +from tqdm import tqdm +import cv2 + +CROP_SCALES = 3 # follow OpenMask3D +args = get_args() + +class CroppedImageDataset(Dataset): + def __init__(self, seq_name_list, frame_id_list, mask_id_list, rgb_path_list, segmentation_path_list, preprocess): + ''' + Given a list of masks, we calculate the open-vocabulary features for each mask. + + Args: + seq_name_list: sequence name for each mask + frame_id_list: frame id for each mask + mask_id_list: mask id for each mask + rgb_path_list: rgb path for each mask + segmentation_path_list: segmentation path for each mask + preprocess: image preprocessing function + ''' + self.seq_name_list = seq_name_list + self.frame_id_list = frame_id_list + self.mask_id_list = mask_id_list + self.preprocess = preprocess + self.rgb_path_list = rgb_path_list + self.segmentation_path_list = segmentation_path_list + + def __len__(self): + return len(self.mask_id_list) + + def __getitem__(self, idx): + def get_cropped_image(mask, rgb): + ''' + Given a mask and an rgb image, we crop the image with CROP_SCALES scales based on the mask. + ''' + def mask2box_multi_level(mask, level, expansion_ratio): + pos = np.where(mask) + top = np.min(pos[0]) + bottom = np.max(pos[0]) + left = np.min(pos[1]) + right = np.max(pos[1]) + + if level == 0: + return left, top, right , bottom + shape = mask.shape + x_exp = int(abs(right - left)*expansion_ratio) * level + y_exp = int(abs(bottom - top)*expansion_ratio) * level + return max(0, left - x_exp), max(0, top - y_exp), min(shape[1], right + x_exp), min(shape[0], bottom + y_exp) + + def crop_image(rgb, mask): + multiscale_cropped_images = [] + for level in range(CROP_SCALES): + left, top, right, bottom = mask2box_multi_level(mask, level, 0.1) + cropped_image = rgb[top:bottom, left:right].copy() + multiscale_cropped_images.append(cropped_image) + return multiscale_cropped_images + + mask = cv2.resize(mask.astype(np.uint8), (rgb.shape[1], rgb.shape[0]), interpolation=cv2.INTER_NEAREST) + multiscale_cropped_images = crop_image(rgb, mask) + return multiscale_cropped_images + + def pad_into_square(image): + width, height = image.size + new_size = max(width, height) + new_image = Image.new("RGB", (new_size, new_size), (255,255,255)) + left = (new_size - width) // 2 + top = (new_size - height) // 2 + new_image.paste(image, (left, top)) + return new_image + + seq_name = self.seq_name_list[idx] + frame_id = self.frame_id_list[idx] + mask_id = self.mask_id_list[idx] + rgb_path = self.rgb_path_list[idx] + segmentation_path = self.segmentation_path_list[idx] + + rgb_image = cv2.imread(rgb_path) + rgb_image = cv2.cvtColor(rgb_image, cv2.COLOR_BGR2RGB) + + segmentation_image = cv2.imread(segmentation_path, cv2.IMREAD_UNCHANGED) + mask = (segmentation_image == mask_id) + cropped_images = get_cropped_image(mask, np.array(rgb_image)) + + input_images = [self.preprocess(pad_into_square(Image.fromarray(cropped_image))) for cropped_image in cropped_images] + input_images = torch.stack(input_images) + return input_images, seq_name, frame_id, mask_id + +def load_clip(): + print(f'[INFO] loading CLIP model...') + model, _, preprocess = open_clip.create_model_and_transforms("ViT-H-14", pretrained="laion2b_s32b_b79k") + model.cuda() + model.eval() + print(f'[INFO]', ' finish loading CLIP model...') + return model, preprocess + +def main(): + model, preprocess = load_clip() + + seq_name_list, frame_id_list, mask_id_list, rgb_path_list, segmentation_path_list = [], [], [], [], [] + feature_dict = {} + for seq_name in args.seq_name_list.split('+'): + args.seq_name = seq_name + dataset = get_dataset(args) + object_dict = np.load(f'{dataset.object_dict_dir}/{args.config}/object_dict.npy', allow_pickle=True).item() + for key, value in object_dict.items(): + mask_list = value['repre_mask_list'] + if len(mask_list) == 0: + continue + for mask_info in mask_list: + seq_name_list.append(seq_name) + frame_id = mask_info[0] + frame_id_list.append(frame_id) + mask_id_list.append(mask_info[1]) + rgb_path, segmentation_path = dataset.get_frame_path(frame_id) + rgb_path_list.append(rgb_path) + segmentation_path_list.append(segmentation_path) + feature_dict[seq_name] = {} + + dataloader = DataLoader(CroppedImageDataset(seq_name_list, frame_id_list, mask_id_list, rgb_path_list, segmentation_path_list, preprocess), batch_size=64, shuffle=False, num_workers=16) + + print('[INFO] extracting features') + for images, seq_names, frame_ids, mask_ids in tqdm(dataloader): + images = images.reshape(-1, 3, 224, 224) + image_input = images.cuda() + with torch.no_grad(): + image_features = model.encode_image(image_input).float() + image_features /= image_features.norm(dim=-1, keepdim=True) + image_features = image_features.cpu().numpy() + for i in range(len(image_features) // CROP_SCALES): + feature_dict[seq_names[i]][f'{frame_ids[i]}_{mask_ids[i]}'] = image_features[CROP_SCALES*i:CROP_SCALES*(i+1)].mean(axis=0) + print('[INFO] finish extracting features') + + for seq_name in args.seq_name_list.split('+'): + args.seq_name = seq_name + dataset = get_dataset(args) + if seq_name in feature_dict: + np.save(os.path.join(dataset.object_dict_dir, f'{args.config}/open-vocabulary_features.npy'), feature_dict[seq_name]) + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/MaskClustering/semantics/itw-ovq.py b/MaskClustering/semantics/itw-ovq.py new file mode 100644 index 0000000000000000000000000000000000000000..f522c9c5ebcf27135c763111ab3c745b677f021c --- /dev/null +++ b/MaskClustering/semantics/itw-ovq.py @@ -0,0 +1,67 @@ +''' + This script is used to generate the semantic labels for the objects in the scene. +''' +from utils.config import get_dataset, get_args +import os +import numpy as np + +def sigmoid(x): + return 1 / (1 + np.exp(-x)) + +def cos_sim(x_cos): + return (1 + x_cos) / 2 + +def main(args): + dataset = get_dataset(args) + total_point_num = dataset.get_scene_points().shape[0] + + label_features_dict = dataset.get_label_features() + label_text_features = np.stack(list(label_features_dict.values())) + descriptions = list(label_features_dict.keys()) + + object_dict = np.load(f'{dataset.object_dict_dir}/{args.config}/object_dict.npy', allow_pickle=True).item() + clip_feature = np.load(f'{dataset.object_dict_dir}/{args.config}/open-vocabulary_features.npy', allow_pickle=True).item() + label2id = dataset.get_label_id()[0] + + num_instance = len(object_dict) + pred_dict = { + "pred_masks": np.zeros((total_point_num, num_instance), dtype=bool), + "pred_score": np.ones(num_instance), + "pred_classes": np.zeros(num_instance, dtype=np.int32), + "pred_descriptions": ['' for _ in range(num_instance)] + } + + # For each object, average the visual features of the representative masks as its object feature. + # Then get semantic label according to the similarity between the object feature and the label text features. + for idx, (key, value) in enumerate(object_dict.items()): + repre_mask_list = value['repre_mask_list'] + if len(repre_mask_list) == 0: + continue + + feature_list = [] + feature_list = [clip_feature[f'{mask_info[0]}_{mask_info[1]}'] for mask_info in repre_mask_list] + feature = np.stack(feature_list) + object_feature = np.mean(feature, axis=0, keepdims=True) + + raw_similarity = np.dot(object_feature, label_text_features.T) + exp_sim = cos_sim(raw_similarity) + prob = exp_sim + probs = np.max(prob, axis=0) + max_label_id = np.argmax(probs) + prob = probs[max_label_id] + pred_dict['pred_score'][idx] = prob + + label_id = label2id[descriptions[max_label_id]] + pred_dict['pred_classes'][idx] = label_id + pred_dict['pred_descriptions'][idx] = descriptions[max_label_id] + + point_ids = object_dict[key]['point_ids'] + binary_mask = np.zeros(total_point_num, dtype=bool) + binary_mask[list(point_ids)] = True + pred_dict['pred_masks'][:, idx] = binary_mask + + np.savez(f'{dataset.object_dict_dir}/prediction.npz', **pred_dict) + +if __name__ == '__main__': + args = get_args() + main(args) \ No newline at end of file diff --git a/MaskClustering/semantics/open-voc_query.py b/MaskClustering/semantics/open-voc_query.py new file mode 100644 index 0000000000000000000000000000000000000000..7771ccf42cdd80aab83bd34b6a2e0233a651bce0 --- /dev/null +++ b/MaskClustering/semantics/open-voc_query.py @@ -0,0 +1,62 @@ +''' + This script is used to generate the semantic labels for the objects in the scene. +''' +from utils.config import get_dataset, get_args +import os +import numpy as np + +def main(args): + dataset = get_dataset(args) + total_point_num = dataset.get_scene_points().shape[0] + + label_features_dict = dataset.get_label_features() + label_text_features = np.stack(list(label_features_dict.values())) + descriptions = list(label_features_dict.keys()) + + object_dict = np.load(f'{dataset.object_dict_dir}/{args.config}/object_dict.npy', allow_pickle=True).item() + clip_feature = np.load(f'{dataset.object_dict_dir}/{args.config}/open-vocabulary_features.npy', allow_pickle=True).item() + label2id = dataset.get_label_id()[0] + + pred_dir = os.path.join('data/prediction', args.config) + os.makedirs(pred_dir, exist_ok=True) + + num_instance = len(object_dict) + pred_dict = { + "pred_masks": np.zeros((total_point_num, num_instance), dtype=bool), + "pred_score": np.ones(num_instance), + "pred_classes" : np.zeros(num_instance, dtype=np.int32) + } + + # For each object, average the visual features of the representative masks as its object feature. + # Then get semantic label according to the similarity between the object feature and the label text features. + for idx, (key, value) in enumerate(object_dict.items()): + repre_mask_list = value['repre_mask_list'] + if len(repre_mask_list) == 0: + continue + + feature_list = [] + feature_list = [clip_feature[f'{mask_info[0]}_{mask_info[1]}'] for mask_info in repre_mask_list] + feature = np.stack(feature_list) + object_feature = np.mean(feature, axis=0, keepdims=True) + + raw_similarity = np.dot(object_feature, label_text_features.T) + exp_sim = np.exp(raw_similarity * 100) + prob = exp_sim / np.sum(exp_sim, axis=1, keepdims=True) + probs = np.max(prob, axis=0) + max_label_id = np.argmax(probs) + prob = probs[max_label_id] + pred_dict['pred_score'][idx] = prob + + label_id = label2id[descriptions[max_label_id]] + pred_dict['pred_classes'][idx] = label_id + + point_ids = object_dict[key]['point_ids'] + binary_mask = np.zeros(total_point_num, dtype=bool) + binary_mask[list(point_ids)] = True + pred_dict['pred_masks'][:, idx] = binary_mask + + np.savez(f'{pred_dir}/{args.seq_name}.npz', **pred_dict) + +if __name__ == '__main__': + args = get_args() + main(args) \ No newline at end of file diff --git a/MaskClustering/semantics/wopen-voc_query.py b/MaskClustering/semantics/wopen-voc_query.py new file mode 100644 index 0000000000000000000000000000000000000000..7a29f81fc1b6fb239794ee4fec3458c8bdf6f563 --- /dev/null +++ b/MaskClustering/semantics/wopen-voc_query.py @@ -0,0 +1,65 @@ +''' + This script is used to generate the semantic labels for the objects in the scene. +''' +from utils.config import get_dataset, get_args +import os +import numpy as np + +def sigmoid(x): + return 1 / (1 + np.exp(-x)) + +def cos_sim(x_cos): + return (1 + x_cos) / 2 + +def main(args): + dataset = get_dataset(args) + total_point_num = dataset.get_scene_points().shape[0] + + label_features_dict = dataset.get_label_features() + label_text_features = np.stack(list(label_features_dict.values())) + descriptions = list(label_features_dict.keys()) + + object_dict = np.load(f'{dataset.object_dict_dir}/{args.config}/object_dict.npy', allow_pickle=True).item() + clip_feature = np.load(f'{dataset.object_dict_dir}/{args.config}/open-vocabulary_features.npy', allow_pickle=True).item() + label2id = dataset.get_label_id()[0] + + num_instance = len(object_dict) + pred_dict = { + "pred_masks": np.zeros((total_point_num, num_instance), dtype=bool), + "pred_score": np.ones(num_instance), + "pred_classes" : np.zeros(num_instance, dtype=np.int32) + } + + # For each object, average the visual features of the representative masks as its object feature. + # Then get semantic label according to the similarity between the object feature and the label text features. + for idx, (key, value) in enumerate(object_dict.items()): + repre_mask_list = value['repre_mask_list'] + if len(repre_mask_list) == 0: + continue + + feature_list = [] + feature_list = [clip_feature[f'{mask_info[0]}_{mask_info[1]}'] for mask_info in repre_mask_list] + feature = np.stack(feature_list) + object_feature = np.mean(feature, axis=0, keepdims=True) + + raw_similarity = np.dot(object_feature, label_text_features.T) + exp_sim = cos_sim(raw_similarity) + prob = exp_sim + probs = np.max(prob, axis=0) + max_label_id = np.argmax(probs) + prob = probs[max_label_id] + pred_dict['pred_score'][idx] = prob + + label_id = label2id[descriptions[max_label_id]] + pred_dict['pred_classes'][idx] = label_id + + point_ids = object_dict[key]['point_ids'] + binary_mask = np.zeros(total_point_num, dtype=bool) + binary_mask[list(point_ids)] = True + pred_dict['pred_masks'][:, idx] = binary_mask + + np.savez(f'{dataset.object_dict_dir}/prediction.npz', **pred_dict) + +if __name__ == '__main__': + args = get_args() + main(args) \ No newline at end of file diff --git a/MaskClustering/splits/matterport3d.txt b/MaskClustering/splits/matterport3d.txt new file mode 100644 index 0000000000000000000000000000000000000000..b80e130bfcf711728b139f3418a7d2374396b0e6 --- /dev/null +++ b/MaskClustering/splits/matterport3d.txt @@ -0,0 +1,8 @@ +2t7WUuJeko7 +ARNzJeq3xxb +WYY7iVyf5p8 +YFuZgdQ5vWj +YVUC4YcDtcY +gxdoqLR6rwA +gYvKGZ5eRqb +RPmz2sHmrrY \ No newline at end of file diff --git a/MaskClustering/splits/scannet.txt b/MaskClustering/splits/scannet.txt new file mode 100644 index 0000000000000000000000000000000000000000..b5bc147739edd80a212fd0f524da0a6c42b85fed --- /dev/null +++ b/MaskClustering/splits/scannet.txt @@ -0,0 +1,312 @@ +scene0011_00 +scene0011_01 +scene0015_00 +scene0019_00 +scene0019_01 +scene0025_00 +scene0025_01 +scene0025_02 +scene0030_00 +scene0030_01 +scene0030_02 +scene0046_00 +scene0046_01 +scene0046_02 +scene0050_00 +scene0050_01 +scene0050_02 +scene0063_00 +scene0064_00 +scene0064_01 +scene0077_00 +scene0077_01 +scene0081_00 +scene0081_01 +scene0081_02 +scene0084_00 +scene0084_01 +scene0084_02 +scene0086_00 +scene0086_01 +scene0086_02 +scene0088_00 +scene0088_01 +scene0088_02 +scene0088_03 +scene0095_00 +scene0095_01 +scene0100_00 +scene0100_01 +scene0100_02 +scene0131_00 +scene0131_01 +scene0131_02 +scene0139_00 +scene0144_00 +scene0144_01 +scene0146_00 +scene0146_01 +scene0146_02 +scene0149_00 +scene0153_00 +scene0153_01 +scene0164_00 +scene0164_01 +scene0164_02 +scene0164_03 +scene0169_00 +scene0169_01 +scene0187_00 +scene0187_01 +scene0193_00 +scene0193_01 +scene0196_00 +scene0203_00 +scene0203_01 +scene0203_02 +scene0207_00 +scene0207_01 +scene0207_02 +scene0208_00 +scene0217_00 +scene0221_00 +scene0221_01 +scene0222_00 +scene0222_01 +scene0231_00 +scene0231_01 +scene0231_02 +scene0246_00 +scene0249_00 +scene0251_00 +scene0256_00 +scene0256_01 +scene0256_02 +scene0257_00 +scene0277_00 +scene0277_01 +scene0277_02 +scene0278_00 +scene0278_01 +scene0300_00 +scene0300_01 +scene0304_00 +scene0307_00 +scene0307_01 +scene0307_02 +scene0314_00 +scene0316_00 +scene0328_00 +scene0329_00 +scene0329_01 +scene0329_02 +scene0334_00 +scene0334_01 +scene0334_02 +scene0338_00 +scene0338_01 +scene0338_02 +scene0342_00 +scene0343_00 +scene0351_00 +scene0351_01 +scene0353_00 +scene0353_01 +scene0353_02 +scene0354_00 +scene0355_00 +scene0355_01 +scene0356_00 +scene0356_01 +scene0356_02 +scene0357_00 +scene0357_01 +scene0377_00 +scene0377_01 +scene0377_02 +scene0378_00 +scene0378_01 +scene0378_02 +scene0382_00 +scene0382_01 +scene0389_00 +scene0406_00 +scene0406_01 +scene0406_02 +scene0412_00 +scene0412_01 +scene0414_00 +scene0423_00 +scene0423_01 +scene0423_02 +scene0426_00 +scene0426_01 +scene0426_02 +scene0426_03 +scene0427_00 +scene0430_00 +scene0430_01 +scene0432_00 +scene0432_01 +scene0435_00 +scene0435_01 +scene0435_02 +scene0435_03 +scene0441_00 +scene0458_00 +scene0458_01 +scene0461_00 +scene0462_00 +scene0474_00 +scene0474_01 +scene0474_02 +scene0474_03 +scene0474_04 +scene0474_05 +scene0488_00 +scene0488_01 +scene0490_00 +scene0494_00 +scene0496_00 +scene0500_00 +scene0500_01 +scene0518_00 +scene0527_00 +scene0535_00 +scene0549_00 +scene0549_01 +scene0550_00 +scene0552_00 +scene0552_01 +scene0553_00 +scene0553_01 +scene0553_02 +scene0558_00 +scene0558_01 +scene0558_02 +scene0559_00 +scene0559_01 +scene0559_02 +scene0565_00 +scene0568_00 +scene0568_01 +scene0568_02 +scene0574_00 +scene0574_01 +scene0574_02 +scene0575_00 +scene0575_01 +scene0575_02 +scene0578_00 +scene0578_01 +scene0578_02 +scene0580_00 +scene0580_01 +scene0583_00 +scene0583_01 +scene0583_02 +scene0591_00 +scene0591_01 +scene0591_02 +scene0593_00 +scene0593_01 +scene0595_00 +scene0598_00 +scene0598_01 +scene0598_02 +scene0599_00 +scene0599_01 +scene0599_02 +scene0606_00 +scene0606_01 +scene0606_02 +scene0607_00 +scene0607_01 +scene0608_00 +scene0608_01 +scene0608_02 +scene0609_00 +scene0609_01 +scene0609_02 +scene0609_03 +scene0616_00 +scene0616_01 +scene0618_00 +scene0621_00 +scene0629_00 +scene0629_01 +scene0629_02 +scene0633_00 +scene0633_01 +scene0643_00 +scene0644_00 +scene0645_00 +scene0645_01 +scene0645_02 +scene0647_00 +scene0647_01 +scene0648_00 +scene0648_01 +scene0651_00 +scene0651_01 +scene0651_02 +scene0652_00 +scene0653_00 +scene0653_01 +scene0655_00 +scene0655_01 +scene0655_02 +scene0658_00 +scene0660_00 +scene0663_00 +scene0663_01 +scene0663_02 +scene0664_00 +scene0664_01 +scene0664_02 +scene0665_00 +scene0665_01 +scene0670_00 +scene0670_01 +scene0671_00 +scene0671_01 +scene0678_00 +scene0678_01 +scene0678_02 +scene0684_00 +scene0684_01 +scene0685_00 +scene0685_01 +scene0685_02 +scene0686_00 +scene0686_01 +scene0686_02 +scene0689_00 +scene0690_00 +scene0690_01 +scene0693_00 +scene0693_01 +scene0693_02 +scene0695_00 +scene0695_01 +scene0695_02 +scene0695_03 +scene0696_00 +scene0696_01 +scene0696_02 +scene0697_00 +scene0697_01 +scene0697_02 +scene0697_03 +scene0699_00 +scene0700_00 +scene0700_01 +scene0700_02 +scene0701_00 +scene0701_01 +scene0701_02 +scene0702_00 +scene0702_01 +scene0702_02 +scene0704_00 +scene0704_01 \ No newline at end of file diff --git a/MaskClustering/splits/scannet200_subset.txt b/MaskClustering/splits/scannet200_subset.txt new file mode 100644 index 0000000000000000000000000000000000000000..73a01bff1c1c1b10847fcb5cb54fc68cde1e7f92 --- /dev/null +++ b/MaskClustering/splits/scannet200_subset.txt @@ -0,0 +1,20 @@ +scene0011_00 +scene0011_01 +scene0015_00 +scene0019_00 +scene0019_01 +scene0025_00 +scene0025_01 +scene0025_02 +scene0030_00 +scene0030_01 +scene0030_02 +scene0046_00 +scene0046_01 +scene0046_02 +scene0050_00 +scene0050_01 +scene0050_02 +scene0063_00 +scene0064_00 +scene0064_01 \ No newline at end of file diff --git a/MaskClustering/splits/scannet_all.txt b/MaskClustering/splits/scannet_all.txt new file mode 100644 index 0000000000000000000000000000000000000000..b5bc147739edd80a212fd0f524da0a6c42b85fed --- /dev/null +++ b/MaskClustering/splits/scannet_all.txt @@ -0,0 +1,312 @@ +scene0011_00 +scene0011_01 +scene0015_00 +scene0019_00 +scene0019_01 +scene0025_00 +scene0025_01 +scene0025_02 +scene0030_00 +scene0030_01 +scene0030_02 +scene0046_00 +scene0046_01 +scene0046_02 +scene0050_00 +scene0050_01 +scene0050_02 +scene0063_00 +scene0064_00 +scene0064_01 +scene0077_00 +scene0077_01 +scene0081_00 +scene0081_01 +scene0081_02 +scene0084_00 +scene0084_01 +scene0084_02 +scene0086_00 +scene0086_01 +scene0086_02 +scene0088_00 +scene0088_01 +scene0088_02 +scene0088_03 +scene0095_00 +scene0095_01 +scene0100_00 +scene0100_01 +scene0100_02 +scene0131_00 +scene0131_01 +scene0131_02 +scene0139_00 +scene0144_00 +scene0144_01 +scene0146_00 +scene0146_01 +scene0146_02 +scene0149_00 +scene0153_00 +scene0153_01 +scene0164_00 +scene0164_01 +scene0164_02 +scene0164_03 +scene0169_00 +scene0169_01 +scene0187_00 +scene0187_01 +scene0193_00 +scene0193_01 +scene0196_00 +scene0203_00 +scene0203_01 +scene0203_02 +scene0207_00 +scene0207_01 +scene0207_02 +scene0208_00 +scene0217_00 +scene0221_00 +scene0221_01 +scene0222_00 +scene0222_01 +scene0231_00 +scene0231_01 +scene0231_02 +scene0246_00 +scene0249_00 +scene0251_00 +scene0256_00 +scene0256_01 +scene0256_02 +scene0257_00 +scene0277_00 +scene0277_01 +scene0277_02 +scene0278_00 +scene0278_01 +scene0300_00 +scene0300_01 +scene0304_00 +scene0307_00 +scene0307_01 +scene0307_02 +scene0314_00 +scene0316_00 +scene0328_00 +scene0329_00 +scene0329_01 +scene0329_02 +scene0334_00 +scene0334_01 +scene0334_02 +scene0338_00 +scene0338_01 +scene0338_02 +scene0342_00 +scene0343_00 +scene0351_00 +scene0351_01 +scene0353_00 +scene0353_01 +scene0353_02 +scene0354_00 +scene0355_00 +scene0355_01 +scene0356_00 +scene0356_01 +scene0356_02 +scene0357_00 +scene0357_01 +scene0377_00 +scene0377_01 +scene0377_02 +scene0378_00 +scene0378_01 +scene0378_02 +scene0382_00 +scene0382_01 +scene0389_00 +scene0406_00 +scene0406_01 +scene0406_02 +scene0412_00 +scene0412_01 +scene0414_00 +scene0423_00 +scene0423_01 +scene0423_02 +scene0426_00 +scene0426_01 +scene0426_02 +scene0426_03 +scene0427_00 +scene0430_00 +scene0430_01 +scene0432_00 +scene0432_01 +scene0435_00 +scene0435_01 +scene0435_02 +scene0435_03 +scene0441_00 +scene0458_00 +scene0458_01 +scene0461_00 +scene0462_00 +scene0474_00 +scene0474_01 +scene0474_02 +scene0474_03 +scene0474_04 +scene0474_05 +scene0488_00 +scene0488_01 +scene0490_00 +scene0494_00 +scene0496_00 +scene0500_00 +scene0500_01 +scene0518_00 +scene0527_00 +scene0535_00 +scene0549_00 +scene0549_01 +scene0550_00 +scene0552_00 +scene0552_01 +scene0553_00 +scene0553_01 +scene0553_02 +scene0558_00 +scene0558_01 +scene0558_02 +scene0559_00 +scene0559_01 +scene0559_02 +scene0565_00 +scene0568_00 +scene0568_01 +scene0568_02 +scene0574_00 +scene0574_01 +scene0574_02 +scene0575_00 +scene0575_01 +scene0575_02 +scene0578_00 +scene0578_01 +scene0578_02 +scene0580_00 +scene0580_01 +scene0583_00 +scene0583_01 +scene0583_02 +scene0591_00 +scene0591_01 +scene0591_02 +scene0593_00 +scene0593_01 +scene0595_00 +scene0598_00 +scene0598_01 +scene0598_02 +scene0599_00 +scene0599_01 +scene0599_02 +scene0606_00 +scene0606_01 +scene0606_02 +scene0607_00 +scene0607_01 +scene0608_00 +scene0608_01 +scene0608_02 +scene0609_00 +scene0609_01 +scene0609_02 +scene0609_03 +scene0616_00 +scene0616_01 +scene0618_00 +scene0621_00 +scene0629_00 +scene0629_01 +scene0629_02 +scene0633_00 +scene0633_01 +scene0643_00 +scene0644_00 +scene0645_00 +scene0645_01 +scene0645_02 +scene0647_00 +scene0647_01 +scene0648_00 +scene0648_01 +scene0651_00 +scene0651_01 +scene0651_02 +scene0652_00 +scene0653_00 +scene0653_01 +scene0655_00 +scene0655_01 +scene0655_02 +scene0658_00 +scene0660_00 +scene0663_00 +scene0663_01 +scene0663_02 +scene0664_00 +scene0664_01 +scene0664_02 +scene0665_00 +scene0665_01 +scene0670_00 +scene0670_01 +scene0671_00 +scene0671_01 +scene0678_00 +scene0678_01 +scene0678_02 +scene0684_00 +scene0684_01 +scene0685_00 +scene0685_01 +scene0685_02 +scene0686_00 +scene0686_01 +scene0686_02 +scene0689_00 +scene0690_00 +scene0690_01 +scene0693_00 +scene0693_01 +scene0693_02 +scene0695_00 +scene0695_01 +scene0695_02 +scene0695_03 +scene0696_00 +scene0696_01 +scene0696_02 +scene0697_00 +scene0697_01 +scene0697_02 +scene0697_03 +scene0699_00 +scene0700_00 +scene0700_01 +scene0700_02 +scene0701_00 +scene0701_01 +scene0701_02 +scene0702_00 +scene0702_01 +scene0702_02 +scene0704_00 +scene0704_01 \ No newline at end of file diff --git a/MaskClustering/splits/scannetpp.txt b/MaskClustering/splits/scannetpp.txt new file mode 100644 index 0000000000000000000000000000000000000000..31753ff1134f1dd5c2be80319e741b00ce819946 --- /dev/null +++ b/MaskClustering/splits/scannetpp.txt @@ -0,0 +1,50 @@ +7b6477cb95 +c50d2d1d42 +cc5237fd77 +acd95847c5 +fb5a96b1a2 +a24f64f7fb +1ada7a0617 +5eb31827b7 +3e8bba0176 +3f15a9266d +21d970d8de +5748ce6f01 +c4c04e6d6c +7831862f02 +bde1e479ad +38d58a7a31 +5ee7c22ba0 +f9f95681fd +3864514494 +40aec5fffa +13c3e046d7 +e398684d27 +a8bf42d646 +45b0dac5e3 +31a2c91c43 +e7af285f7d +286b55a2bf +7bc286c1b6 +f3685d06a9 +b0a08200c9 +825d228aec +a980334473 +f2dc06b1d2 +5942004064 +25f3b7a318 +bcd2436daf +f3d64c30f8 +0d2ee665be +3db0a1c8f3 +ac48a9b736 +c5439f4607 +578511c8a9 +d755b3d9d8 +99fa5c25e1 +09c1414f1b +5f99900f09 +9071e139d9 +6115eddb86 +27dd4da69e +c49a8c6cff \ No newline at end of file diff --git a/MaskClustering/third_party/Entity b/MaskClustering/third_party/Entity new file mode 160000 index 0000000000000000000000000000000000000000..6e7e13ac91ef508088e1b848167c01f19b00b512 --- /dev/null +++ b/MaskClustering/third_party/Entity @@ -0,0 +1 @@ +Subproject commit 6e7e13ac91ef508088e1b848167c01f19b00b512 diff --git a/MaskClustering/third_party/detectron2 b/MaskClustering/third_party/detectron2 new file mode 160000 index 0000000000000000000000000000000000000000..536dc9d527074e3b15df5f6677ffe1f4e104a4ab --- /dev/null +++ b/MaskClustering/third_party/detectron2 @@ -0,0 +1 @@ +Subproject commit 536dc9d527074e3b15df5f6677ffe1f4e104a4ab diff --git a/MaskClustering/utils/clean_all_output.py b/MaskClustering/utils/clean_all_output.py new file mode 100644 index 0000000000000000000000000000000000000000..a65ca15a4f02c8d2a77b750e204b06b8112d030b --- /dev/null +++ b/MaskClustering/utils/clean_all_output.py @@ -0,0 +1,43 @@ +''' + Clean all output files produced by this project. +''' + +from utils.config import get_dataset, get_args +import os +from tqdm import tqdm + +def delete_files(dataset): + dirs_to_be_deleted = [f'{dataset.root}/output'] + for dir_to_be_deleted in dirs_to_be_deleted: + if os.path.exists(dir_to_be_deleted): + os.system(f'rm -r {dir_to_be_deleted}') + +def main(dataset): + delete_files(dataset) + +def process_one_dataset(split_path, args): + with open(split_path, 'r') as f: + seq_name_list = f.readlines() + seq_name_list = [seq_name.strip() for seq_name in seq_name_list] + for seq_name in tqdm(seq_name_list): + args.seq_name = seq_name + dataset = get_dataset(args) + main(dataset) + +if __name__ == '__main__': + args = get_args() + + # ScanNet + split_path = 'splits/scannet.txt' + args.dataset = 'scannet' + process_one_dataset(split_path, args) + + # # ScanNet++ + # split_path = 'splits/scannetpp.txt' + # args.dataset = 'scannetpp' + # process_one_dataset(split_path, args) + + # # MatterPort3d + # split_path = 'splits/matterport3d.txt' + # args.dataset = 'matterport3d' + # process_one_dataset(split_path, args) \ No newline at end of file diff --git a/MaskClustering/utils/config.py b/MaskClustering/utils/config.py new file mode 100644 index 0000000000000000000000000000000000000000..f766e4e56c1483f934ec939eb4e081c38b9397f6 --- /dev/null +++ b/MaskClustering/utils/config.py @@ -0,0 +1,100 @@ +import argparse +from dataset.scannet import ITWDataset, ScanNetDataset, ScanNet18Dataset, ARKitDataset, ScannetPP2Dataset, ScanNet20Dataset, WildDataset, ITWDataset +from dataset.matterport import MatterportDataset +from dataset.scannetpp import ScanNetPPDataset +from dataset.demo import DemoDataset +import json + +def update_args(args): + config = args.config + config_file = config + if config in ['scannet', 'scannet18']: + config_file = 'scannet' + if config in ['scannetpp_v2_dust3r_posed', 'scannetpp_v2_dust3r_unposed']: + config_file = config + config_path = f'/home/jovyan/users/bulat/workspace/3drec/Indoor/MaskClustering/configs/{config_file}.json' + with open(config_path, 'r') as f: + config_data = json.load(f) + for key in config_data: + setattr(args, key, config_data[key]) + return args + +def get_args(): + parser = argparse.ArgumentParser() + parser.add_argument('--seq_name', type=str) + parser.add_argument('--seq_name_list', type=str) + parser.add_argument('--config', type=str, default='scannet') + parser.add_argument('--debug', action="store_true") + parser.add_argument('--root', type=str) + parser.add_argument('-d', '--devices', type=int, nargs='+', default=[0, 1, 2, 3]) + + args = parser.parse_args() + args = update_args(args) + return args + +def get_dataset(args): + if args.dataset == 'scannet': + dataset = ScanNetDataset(args.seq_name) + elif args.dataset == 'scannetpp_dust3r_filtered_depth': + dataset = ScannetPP2Dataset(args.seq_name, root='data/scannetpp_dust3r_filtered_depth') + elif args.dataset == 'wild': + dataset = WildDataset(args.seq_name, root=args.root) + elif args.dataset == 'itw': + dataset = ITWDataset(args.seq_name, root='data/itw') + elif args.dataset == 'scannetpp_dust3r_posed': + dataset = ScannetPP2Dataset(args.seq_name, root='data/scannetpp_dust3r_posed') + elif args.dataset == 'scannetpp_dust3r_unposed': + dataset = ScannetPP2Dataset(args.seq_name, root='data/scannetpp_dust3r_unposed') + elif args.dataset == 'scannetpp_v2_dust3r_posed': + dataset = ScannetPP2Dataset(args.seq_name, root='data/scannetpp_v2_dust3r_posed') + elif args.dataset == 'scannetpp_v2_dust3r_unposed': + dataset = ScannetPP2Dataset(args.seq_name, root='data/scannetpp_v2_dust3r_unposed') + elif args.dataset == 'scannetpp_mapanything_posed': + dataset = ScannetPP2Dataset(args.seq_name, root='data/scannetpp_mapanything_posed') + elif args.dataset == 'scannet_dust3r_posed': + dataset = ScanNet18Dataset(args.seq_name, root='data/scannet_dust3r_posed') + elif args.dataset == 'scannet_dust3r_unposed': + dataset = ScanNet18Dataset(args.seq_name, root='data/scannet_dust3r_unposed') + elif args.dataset == 'scannet_dust3r_posed_15': + dataset = ScanNet20Dataset(args.seq_name, root='data/scannet_dust3r_posed_15') + elif args.dataset == 'scannet_dust3r_posed_25': + dataset = ScanNet20Dataset(args.seq_name, root='data/scannet_dust3r_posed_25') + elif args.dataset == 'scannet_dust3r_posed_35': + dataset = ScanNet20Dataset(args.seq_name, root='data/scannet_dust3r_posed_35') + elif args.dataset == 'scannet_dust3r_posed_45': + dataset = ScanNet20Dataset(args.seq_name, root='data/scannet_dust3r_posed_45') + elif args.dataset == 'scannet_dust3r_posed_45_andrey': + dataset = ScanNet20Dataset(args.seq_name, root='data/scannet_dust3r_posed_45_andrey') + elif args.dataset == 'scannet_dust3r_posed_45_bulat': + dataset = ScanNet20Dataset(args.seq_name, root='data/scannet_dust3r_posed_45_bulat') + elif args.dataset == 'scannet_dust3r_posed_35_bulat': + dataset = ScanNet20Dataset(args.seq_name, root='data/scannet_dust3r_posed_35_bulat') + elif args.dataset == 'scannet_dust3r_unposed_15': + dataset = ScanNet20Dataset(args.seq_name, root='data/scannet_dust3r_unposed_15') + elif args.dataset == 'scannet_dust3r_unposed_25': + dataset = ScanNet20Dataset(args.seq_name, root='data/scannet_dust3r_unposed_25') + elif args.dataset == 'scannet_dust3r_unposed_35': + dataset = ScanNet20Dataset(args.seq_name, root='data/scannet_dust3r_unposed_35') + elif args.dataset == 'scannet_dust3r_unposed_45': + dataset = ScanNet20Dataset(args.seq_name, root='data/scannet_dust3r_unposed_45') + elif args.dataset == 'arkit_dust3r_posed': + dataset = ARKitDataset(args.seq_name, root='data/arkit_dust3r_posed') + elif args.dataset == 'arkit_gt': + dataset = ARKitDataset(args.seq_name, root='data/arkit_gt') + elif args.dataset == 'arkit_gt_train': + dataset = ARKitDataset(args.seq_name, root='data/arkit_gt_train') + elif args.dataset == 'arkit_vggt': + dataset = ARKitDataset(args.seq_name, root='data/arkit_vggt') + elif args.dataset == 'scannet18': + dataset = ScanNet18Dataset(args.seq_name) + elif args.dataset == 'scannetpp': + dataset = ScanNetPPDataset(args.seq_name) + elif args.dataset == 'matterport3d': + dataset = MatterportDataset(args.seq_name) + elif args.dataset == 'demo': + dataset = DemoDataset(args.seq_name) + else: + print(args.dataset) + raise NotImplementedError + return dataset + diff --git a/MaskClustering/utils/geometry.py b/MaskClustering/utils/geometry.py new file mode 100644 index 0000000000000000000000000000000000000000..37d6b6d5545cb14bdd2a58e3196e7430c732bf66 --- /dev/null +++ b/MaskClustering/utils/geometry.py @@ -0,0 +1,35 @@ +import numpy as np + +def judge_bbox_overlay(bbox_1, bbox_2): + for i in range(3): + if bbox_1[0][i] > bbox_2[1][i] or bbox_2[0][i] > bbox_1[1][i]: + return False + return True + +def denoise(pcd): + labels = np.array(pcd.cluster_dbscan(eps=0.04, min_points=4)) + 1 # -1 for noise + mask = np.ones(len(labels), dtype=bool) + count = np.bincount(labels) + + # remove component with less than 20% points + for i in range(len(count)): + if count[i] < 0.2 * len(labels): + mask[labels == i] = False + + remain_index = np.where(mask)[0] + pcd = pcd.select_by_index(remain_index) + + pcd, index = pcd.remove_statistical_outlier(nb_neighbors=20, std_ratio=2.0) + remain_index = remain_index[index] + return pcd, remain_index + +def filter_boundary(depth, delta=0.05): + remove_mask = np.zeros(depth.shape).astype(bool) + delta_depth_1 = np.abs(depth[1:, :] - depth[:-1, :]) + delta_depth_2 = np.abs(depth[:, 1:] - depth[:, :-1]) + remove_mask[1:, :] = remove_mask[1:, :] | (delta_depth_1 > delta) + remove_mask[:-1, :] = remove_mask[:-1, :] | (delta_depth_1 > delta) + remove_mask[:, 1:] = remove_mask[:, 1:] | (delta_depth_2 > delta) + remove_mask[:, :-1] = remove_mask[:, :-1] | (delta_depth_2 > delta) + depth[remove_mask] = 0 + return depth \ No newline at end of file diff --git a/MaskClustering/utils/mask_backprojection.py b/MaskClustering/utils/mask_backprojection.py new file mode 100644 index 0000000000000000000000000000000000000000..4d754f2ea93f7e2a9487923b0b4fb5393544ee11 --- /dev/null +++ b/MaskClustering/utils/mask_backprojection.py @@ -0,0 +1,136 @@ +import numpy as np +from pytorch3d.ops import ball_query +import torch +import open3d as o3d +from utils.geometry import denoise +from torch.nn.utils.rnn import pad_sequence + +COVERAGE_THRESHOLD = 0.3 +DISTANCE_THRESHOLD = 0.03 +FEW_POINTS_THRESHOLD = 25 +DEPTH_TRUNC = 20 +BBOX_EXPAND = 0.1 + + +def backproject(depth, intrinisc_cam_parameters, extrinsics): + """ + convert color and depth to view pointcloud + """ + depth = o3d.geometry.Image(depth) + pcld = o3d.geometry.PointCloud.create_from_depth_image(depth, intrinisc_cam_parameters, depth_scale=1, depth_trunc=DEPTH_TRUNC) + pcld.transform(extrinsics) + return pcld + + +def get_neighbor(valid_points, scene_points, lengths_1, lengths_2): + _, neighbor_in_scene_pcld, _ = ball_query(valid_points, scene_points, lengths_1, lengths_2, K=20, radius=DISTANCE_THRESHOLD, return_nn=False) + return neighbor_in_scene_pcld + + +def get_depth_mask(depth): + depth_tensor = torch.from_numpy(depth).cuda() + depth_mask = torch.logical_and(depth_tensor > 0, depth_tensor < DEPTH_TRUNC).reshape(-1) + return depth_mask + + +def crop_scene_points(mask_points, scene_points): + x_min, x_max = torch.min(mask_points[:, 0]), torch.max(mask_points[:, 0]) + y_min, y_max = torch.min(mask_points[:, 1]), torch.max(mask_points[:, 1]) + z_min, z_max = torch.min(mask_points[:, 2]), torch.max(mask_points[:, 2]) + + selected_point_mask = (scene_points[:, 0] > x_min) & (scene_points[:, 0] < x_max) & (scene_points[:, 1] > y_min) & (scene_points[:, 1] < y_max) & (scene_points[:, 2] > z_min) & (scene_points[:, 2] < z_max) + selected_point_ids = torch.where(selected_point_mask)[0] + cropped_scene_points = scene_points[selected_point_ids] + return cropped_scene_points, selected_point_ids + + +def turn_mask_to_point(dataset, scene_points, mask_image, frame_id): + intrinisc_cam_parameters = dataset.get_intrinsics(frame_id) + extrinsics = dataset.get_extrinsic(frame_id) + if np.sum(np.isinf(extrinsics)) > 0: + return {}, [], set() + + depth = dataset.get_depth(frame_id) + depth_mask = get_depth_mask(depth) + + mask_image = torch.from_numpy(mask_image).cuda().reshape(-1) + ids = torch.unique(mask_image).cpu().numpy() + ids.sort() + + + colored_pcld = backproject(depth, intrinisc_cam_parameters, extrinsics) + view_points = np.asarray(colored_pcld.points) + + mask_points_list = [] + mask_points_num_list = [] + scene_points_list = [] + scene_points_num_list = [] + selected_point_ids_list = [] + initial_valid_mask_ids = [] + for mask_id in ids: + if mask_id == 0: + continue + segmentation = mask_image == mask_id + valid_mask = segmentation[depth_mask].cpu().numpy() + + mask_pcld = o3d.geometry.PointCloud() + try: + mask_points = view_points[valid_mask] + except IndexError: + print(f"Error in mask_id: {mask_id}") + continue + if len(mask_points) < FEW_POINTS_THRESHOLD: + continue + mask_pcld.points = o3d.utility.Vector3dVector(mask_points) + + mask_pcld = mask_pcld.voxel_down_sample(voxel_size=DISTANCE_THRESHOLD) + mask_pcld, _ = denoise(mask_pcld) + mask_points = np.asarray(mask_pcld.points) + if len(mask_points) < FEW_POINTS_THRESHOLD: + continue + + mask_points = torch.tensor(mask_points).float().cuda() + cropped_scene_points, selected_point_ids = crop_scene_points(mask_points, scene_points) + initial_valid_mask_ids.append(mask_id) + mask_points_list.append(mask_points) + scene_points_list.append(cropped_scene_points) + mask_points_num_list.append(len(mask_points)) + scene_points_num_list.append(len(cropped_scene_points)) + selected_point_ids_list.append(selected_point_ids) + + if len(initial_valid_mask_ids) == 0: + return {}, [], [] + mask_points_tensor = pad_sequence(mask_points_list, batch_first=True, padding_value=0) + scene_points_tensor = pad_sequence(scene_points_list, batch_first=True, padding_value=0) + + lengths_1 = torch.tensor(mask_points_num_list).cuda() + lengths_2 = torch.tensor(scene_points_num_list).cuda() + neighbor_in_scene_pcld = get_neighbor(mask_points_tensor, scene_points_tensor, lengths_1, lengths_2) + + valid_mask_ids = [] + mask_info = {} + frame_point_ids = set() + + for i, mask_id in enumerate(initial_valid_mask_ids): + mask_neighbor = neighbor_in_scene_pcld[i] # P, 20 + mask_point_num = mask_points_num_list[i] # Pi + mask_neighbor = mask_neighbor[:mask_point_num] # Pi, 20 + + valid_neighbor = mask_neighbor != -1 # Pi, 20 + neighbor = torch.unique(mask_neighbor[valid_neighbor]) + neighbor_in_complete_scene_points = selected_point_ids_list[i][neighbor].cpu().numpy() + coverage = torch.any(valid_neighbor, dim=1).sum().item() / mask_point_num + + if coverage < COVERAGE_THRESHOLD: + continue + valid_mask_ids.append(mask_id) + mask_info[mask_id] = set(neighbor_in_complete_scene_points) + frame_point_ids.update(mask_info[mask_id]) + + return mask_info, valid_mask_ids, list(frame_point_ids) + + +def frame_backprojection(dataset, scene_points, frame_id): + mask_image = dataset.get_segmentation(frame_id, align_with_depth=True) + mask_info, _, frame_point_ids = turn_mask_to_point(dataset, scene_points, mask_image, frame_id) + return mask_info, frame_point_ids \ No newline at end of file diff --git a/MaskClustering/utils/post_process.py b/MaskClustering/utils/post_process.py new file mode 100644 index 0000000000000000000000000000000000000000..b1cb244076a85416d51fd089c7846fb1ac4ac90e --- /dev/null +++ b/MaskClustering/utils/post_process.py @@ -0,0 +1,204 @@ +import numpy as np +import os +import torch +from utils.geometry import judge_bbox_overlay +import traceback + +def merge_overlapping_objects(total_point_ids_list, total_bbox_list, total_mask_list, overlapping_ratio): + ''' + Merge objects that have larger than 0.8 overlapping ratio. + ''' + total_object_num = len(total_point_ids_list) + invalid_object = np.zeros(total_object_num, dtype=bool) + + for i in range(total_object_num): + if invalid_object[i]: + continue + point_ids_i = set(total_point_ids_list[i]) + bbox_i = total_bbox_list[i] + for j in range(i+1, total_object_num): + if invalid_object[j]: + continue + point_ids_j = set(total_point_ids_list[j]) + bbox_j = total_bbox_list[j] + if judge_bbox_overlay(bbox_i, bbox_j): + intersect = len(point_ids_i.intersection(point_ids_j)) + if intersect / len(point_ids_i) > overlapping_ratio: + invalid_object[i] = True + elif intersect / len(point_ids_j) > overlapping_ratio: + invalid_object[j] = True + + valid_point_ids_list = [] + valid_pcld_mask_list = [] + for i in range(total_object_num): + if not invalid_object[i]: + valid_point_ids_list.append(total_point_ids_list[i]) + valid_pcld_mask_list.append(total_mask_list[i]) + return valid_point_ids_list, valid_pcld_mask_list + + +def filter_point(point_frame_matrix, node, pcld_list, point_ids_list, mask_point_clouds, frame_list, args): + ''' + Following OVIR-3D, we filter the points that hardly appear in this cluster (node), i.e. the detection ratio is lower than a threshold. + Specifically, detection ratio = #frames that the point appears in this cluster (node) / #frames that the point appears in the whole video. + ''' + def count_point_appears_in_video(point_frame_matrix, point_ids_list, node_global_frame_id_list): + ''' + For all points in the cluster, compute #frames that the point appears in the whole video. + Initialize #frames that the point appears in this cluster as 0. + ''' + point_appear_in_video_nums, point_appear_in_node_matrixs = [], [] + for point_ids in point_ids_list: + point_appear_in_video_matrix = point_frame_matrix[point_ids, ] + point_appear_in_video_matrix = point_appear_in_video_matrix[:, node_global_frame_id_list] + point_appear_in_video_nums.append(np.sum(point_appear_in_video_matrix, axis=1)) + + point_appear_in_node_matrix = np.zeros_like(point_appear_in_video_matrix, dtype=bool) # initialize as False + point_appear_in_node_matrixs.append(point_appear_in_node_matrix) + return point_appear_in_video_nums, point_appear_in_node_matrixs + + def count_point_appears_in_node(mask_list, node_frame_id_list, point_ids_list, mask_point_clouds, point_appear_in_node_matrixs): + ''' + Fillin the point_appear_in_node_matrixs by iterating the masks in this cluster (node). + Meanwhile, since we split the disconnected point cloud into different objects, we also decide which object this mask belongs to. + Besides, for each mask, we compute the coverage of this mask of the object it belongs to for furture use in OpenMask3D. + ''' + object_mask_list = [[] for _ in range(len(point_ids_list))] + + for frame_id, mask_id in mask_list: + frame_id_in_list = np.where(node_frame_id_list == frame_id)[0][0] + mask_point_ids = list(mask_point_clouds[f'{frame_id}_{mask_id}']) + + object_id_with_largest_intersect, largest_intersect, coverage = -1, 0, 0 + for i, point_ids in enumerate(point_ids_list): + point_ids_within_object = np.where(np.isin(point_ids, mask_point_ids))[0] + point_appear_in_node_matrixs[i][point_ids_within_object, frame_id_in_list] = True + if len(point_ids_within_object) > largest_intersect: + object_id_with_largest_intersect, largest_intersect = i, len(point_ids_within_object) + coverage = len(point_ids_within_object) / len(point_ids) + if largest_intersect == 0: + continue + object_mask_list[object_id_with_largest_intersect] += [(frame_id, mask_id, coverage)] + return object_mask_list, point_appear_in_node_matrixs + + node_global_frame_id_list = torch.where(node.visible_frame)[0].cpu().numpy() + node_frame_id_list = np.array(frame_list)[node_global_frame_id_list] + mask_list = node.mask_list + + point_appear_in_video_nums, point_appear_in_node_matrixs = count_point_appears_in_video(point_frame_matrix, point_ids_list, node_global_frame_id_list) + object_mask_list, point_appear_in_node_matrixs = count_point_appears_in_node(mask_list, node_frame_id_list, point_ids_list, mask_point_clouds, point_appear_in_node_matrixs) + + # filter points + filtered_point_ids, filtered_mask_list, filtered_bbox_list = [], [], [] + for i, (point_appear_in_video_num, point_appear_in_node_matrix) in enumerate(zip(point_appear_in_video_nums, point_appear_in_node_matrixs)): + detection_ratio = np.sum(point_appear_in_node_matrix, axis=1) / (point_appear_in_video_num + 1e-6) + valid_point_ids = np.where(detection_ratio > args.point_filter_threshold)[0] + if len(valid_point_ids) == 0 or len(object_mask_list[i]) < 2: + continue + filtered_point_ids.append(point_ids_list[i][valid_point_ids]) + filtered_bbox_list.append([np.amin(pcld_list[i].points, axis=0), np.amax(pcld_list[i].points, axis=0)]) + filtered_mask_list.append(object_mask_list[i]) + return filtered_point_ids, filtered_bbox_list, filtered_mask_list + + +def dbscan_process(pcld, point_ids, DBSCAN_THRESHOLD=0.1): + ''' + Following OVIR-3D, we use DBSCAN to split the disconnected point cloud into different objects. + ''' + + labels = np.array(pcld.cluster_dbscan(eps=DBSCAN_THRESHOLD, min_points=4)) + 1 # -1 for noise + count = np.bincount(labels) + + # split disconnected point cloud into different objects + pcld_list, point_ids_list = [], [] + pcld_ids_list = np.array(point_ids) + for i in range(len(count)): + remain_index = np.where(labels == i)[0] + if len(remain_index) == 0: + continue + new_pcld = pcld.select_by_index(remain_index) + point_ids = pcld_ids_list[remain_index] + pcld_list.append(new_pcld) + point_ids_list.append(point_ids) + return pcld_list, point_ids_list + + +def find_represent_mask(mask_info_list): + mask_info_list.sort(key=lambda x: x[2], reverse=True) + return mask_info_list[:5] + + +def export_class_agnostic_mask(args, class_agnostic_mask_list): + config = args.config + + pred_dir = os.path.join('data/prediction', config) + os.makedirs(pred_dir, exist_ok=True) + + num_instance = len(class_agnostic_mask_list) + try: + pred_masks = np.stack(class_agnostic_mask_list, axis=1) + except Exception as e: + print(f"class_agnostic_mask_list {class_agnostic_mask_list} has wrong shape") + traceback.print_exc() + return + + pred_dict = { + "pred_masks": pred_masks, + "pred_score": np.ones(num_instance), + "pred_classes" : np.zeros(num_instance, dtype=np.int32) + } + class_agnostic_pred_dir = os.path.join('data/prediction', config + '_class_agnostic') + os.makedirs(class_agnostic_pred_dir, exist_ok=True) + np.savez(os.path.join(class_agnostic_pred_dir, f'{args.seq_name}.npz'), **pred_dict) + return + + +def export(dataset, total_point_ids_list, total_mask_list, args): + ''' + Export class agnostic masks in standard evaluation format + and object dict with corresponding mask lists for semantic instance segmentation. + Node that after clustering, a node = a cluster of masks = an object. + ''' + total_point_num = dataset.get_scene_points().shape[0] + class_agnostic_mask_list = [] + object_dict = {} + for i, (point_ids, mask_list) in enumerate(zip(total_point_ids_list, total_mask_list)): + object_dict[i] = { + 'point_ids': point_ids, + 'mask_list': mask_list, + 'repre_mask_list': find_represent_mask(mask_list), + } + binary_mask = np.zeros(total_point_num, dtype=bool) + binary_mask[list(point_ids)] = True + class_agnostic_mask_list.append(binary_mask) + + export_class_agnostic_mask(args, class_agnostic_mask_list) + + os.makedirs(os.path.join(dataset.object_dict_dir, args.config), exist_ok=True) + np.save(os.path.join(dataset.object_dict_dir, args.config, 'object_dict.npy'), object_dict, allow_pickle=True) + + +def post_process(dataset, node_list, mask_point_clouds, scene_points, point_frame_matrix, frame_list, args): + if args.debug: + print('start exporting') + + + # For each cluster, we follow OVIR-3D to i) use DBScan to split the disconnected point cloud into different objects + # ii) filter the points that hardly appear within this cluster, i.e. the detection ratio is lower than a threshold + total_point_ids_list, total_bbox_list, total_mask_list = [], [], [] + for node in (node_list): + if len(node.mask_list) < 2: # objects merged from less than 2 masks are ignored + continue + + pcld, point_ids = node.get_point_cloud(scene_points) + pcld_list, point_ids_list = dbscan_process(pcld, point_ids) # split the disconnected point cloud into different objects + point_ids_list, bbox_list, mask_list = filter_point(point_frame_matrix, node, pcld_list, point_ids_list, mask_point_clouds, frame_list, args) + + total_point_ids_list.extend(point_ids_list) + total_bbox_list.extend(bbox_list) + total_mask_list.extend(mask_list) + + # merge objects that have larger than 0.8 overlapping ratio + total_point_ids_list, total_mask_list = merge_overlapping_objects(total_point_ids_list, total_bbox_list, total_mask_list, overlapping_ratio=0.8) + export(dataset, total_point_ids_list, total_mask_list, args) + return \ No newline at end of file diff --git a/MaskClustering/visualize/vis_mask.py b/MaskClustering/visualize/vis_mask.py new file mode 100644 index 0000000000000000000000000000000000000000..ec4134fd3a5ec576a28ace4c05ceec59618a9832 --- /dev/null +++ b/MaskClustering/visualize/vis_mask.py @@ -0,0 +1,51 @@ +from utils.config import get_args, get_dataset +import os +import cv2 +import numpy as np + +def create_colormap(): + colormap = np.zeros((256, 3), dtype=int) + ind = np.arange(256, dtype=int) + + for shift in reversed(range(8)): + for channel in range(3): + colormap[:, channel] |= ((ind >> channel) & 1) << shift + ind >>= 3 + + return colormap + +def main(dataset, vis_dir, frame_id): + segmentation_image = dataset.get_segmentation(frame_id) + color_segmentation = np.zeros((segmentation_image.shape[0], segmentation_image.shape[1], 3), dtype=np.uint8) + mask_ids = np.unique(segmentation_image) + mask_ids.sort() + + text_list, text_center_list = [], [] + for mask_id in mask_ids: + if mask_id == 0: + continue + color_segmentation[segmentation_image == mask_id] = colormap[mask_id] + mask_pos = np.where(segmentation_image == mask_id) + mask_center = (int(np.mean(mask_pos[1])), int(np.mean(mask_pos[0]))) + text_list.append(str(mask_id)) + text_center_list.append(mask_center) + + for text, text_center in zip(text_list, text_center_list): + cv2.putText(color_segmentation, text, text_center, cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 0), 2) + + raw_rgb = dataset.get_rgb(frame_id, change_color=False) + concatenate_image = np.concatenate((raw_rgb, color_segmentation), axis=1) + concatenate_image = cv2.resize(concatenate_image, (concatenate_image.shape[1] // 2, concatenate_image.shape[0] // 2)) + cv2.imwrite(os.path.join(vis_dir, str(frame_id) + '.png'), concatenate_image) + return + +if __name__ == '__main__': + args = get_args() + dataset = get_dataset(args) + colormap = create_colormap() + vis_dir = os.path.join(dataset.segmentation_dir, '..', 'vis_mask') + os.makedirs(vis_dir, exist_ok=True) + frame_list = dataset.get_frame_list(args.step) + for frame_id in frame_list: + main(dataset, vis_dir, frame_id) + break \ No newline at end of file diff --git a/MaskClustering/visualize/vis_scene.py b/MaskClustering/visualize/vis_scene.py new file mode 100644 index 0000000000000000000000000000000000000000..5ace0705267d35d294432c868bdde48d851a7c07 --- /dev/null +++ b/MaskClustering/visualize/vis_scene.py @@ -0,0 +1,77 @@ +import numpy as np +import trimesh +from utils.config import get_dataset, get_args +import os + +# Since there are hundreds of objects in the scene, assigning visually distinguishable colors to each object is difficult. You can change the random seed to check if the two objects are actually segmented apart. +np.random.seed(4) + +def vis_one_object(point_ids, scene_points): + points = scene_points[point_ids] + color = (np.random.rand(3) * 0.7 + 0.3) * 255 + colors = np.tile(color, (points.shape[0], 1)) + return point_ids, points, colors, color, np.mean(points, axis=0) + + +def main(args): + dataset = get_dataset(args) + + # Загружаем меш/облако точек используя trimesh + geometry = trimesh.load(dataset.mesh_path) + scene_points = np.array(geometry.vertices) + # scene_points = scene_points - np.mean(scene_points, axis=0) + + # Исходные цвета + if hasattr(geometry.visual, 'vertex_colors'): + scene_colors = geometry.visual.vertex_colors[:, :3].astype(float) + else: + scene_colors = np.ones((len(scene_points), 3)) * 200 # Default color if not available + + # Тон-маппинг для осветления цветов + scene_colors = np.power(scene_colors/255, 1/2.2) * 255 + + # Создаем массивы для хранения меток экземпляров и их цветов + instance_ids = np.zeros(len(scene_points), dtype=np.int32) + instance_colors = np.zeros_like(scene_colors) + + pred = np.load(f'data/prediction/scannet_dust3r_posed_class_agnostic/{args.seq_name}.npz') + # pred = np.load(f'data/prediction/{args.config}/{args.seq_name}.npz') + + masks = pred['pred_masks'] + num_instances = masks.shape[1] + + # Задаем фоновый цвет для точек без инстанса (серый) + background_color = np.array([128, 128, 128]) + instance_colors[:] = background_color # Устанавливаем фоновый цвет для всех точек + + for idx in range(num_instances): + mask = masks[:, idx] + point_ids = np.where(mask)[0] + + point_ids, points, colors, label_color, center = vis_one_object(point_ids, scene_points) + instance_colors[point_ids] = label_color + instance_ids[point_ids] = idx + 1 # Присваиваем ID экземпляра (начиная с 1) + + # Создаем новое облако точек с цветами инстансов + cloud = trimesh.PointCloud( + vertices=scene_points, + colors=instance_colors.astype(np.uint8) # Используем цвета инстансов вместо исходных + ) + + # Добавляем ID инстансов как атрибут вершин + cloud.metadata['instance_ids'] = instance_ids + + # Создаем директорию для сохранения, если она не существует + os.makedirs(f'data/ply_output', exist_ok=True) + + # Сохраняем в PLY формат + output_path = f'data/ply_output/{args.seq_name}_segmented.ply' + cloud.export(output_path) + + print(f"Сегментированное облако точек сохранено в {output_path}") + print(f"Найдено {num_instances} объектов") + + +if __name__ == '__main__': + args = get_args() + main(args) \ No newline at end of file diff --git a/mvp.py b/mvp.py index c995c945ce1774772531dcabc249d33882eb658e..c28329b6746a3c8fbccaa24a441715e2a5eb19a3 100644 --- a/mvp.py +++ b/mvp.py @@ -414,7 +414,7 @@ def reconstruct( log_msg = f"Reconstruction Success ({len(all_files)} frames). Waiting for visualization." os.system(f"python {MK_PATH}/third_party/detectron2/projects/CropFormer/demo_cropformer/mask_predict.py \ --config-file {MK_PATH}/third_party/detectron2/projects/CropFormer/configs/entityv2/entity_segmentation/mask2former_hornet_3x.yaml \ ---root /home/jovyan/users/bulat/workspace/3drec/vggt/temp/input/ --image_path_pattern images/*.jpg --dataset arkit_gt \ +--root temp/input/ --image_path_pattern images/*.jpg --dataset arkit_gt \ --seq_name_list {os.path.basename(target_dir)} --opts MODEL.WEIGHTS \ {MK_PATH}/Mask2Former_hornet_3x_576d0b.pth") os.system(f"python {MK_PATH}/main.py --config wild --root /home/jovyan/users/bulat/workspace/3drec/vggt/temp/input --seq_name_list {os.path.basename(target_dir)}")