import pickle from pathlib import Path import torch import cv2 import numpy as np import os import open3d as o3d from tqdm import tqdm from tqdm.contrib.concurrent import process_map, thread_map from rec_utils.datasets import ARKitDataset, VGGTDataset from rec_utils.aligner import build_aligner_1p1d from rec_utils.datasets.arkit.utils import rotate_image def process_scene(scene_params, target_depth_shape=None): images = scene_params["image_files"] poses = scene_params["poses"] depths = scene_params["depths"] Ks = scene_params["Ks"] pts3d = scene_params["pts3d"] im_confs = scene_params["im_conf"] print(scene_params.keys()) im_shapes = scene_params["imshapes"] im_shape = im_shapes[0] image_hw = cv2.imread(images[0]).shape[:2] image_scale = np.ones((3, 3)) image_scale[0] *= image_hw[1] / im_shape[1] image_scale[1] *= image_hw[0] / im_shape[0] if target_depth_shape is None: target_depth_shape = image_hw depth_scale = np.ones((3, 3)) depth_scale[0] *= target_depth_shape[1] / im_shape[1] depth_scale[1] *= target_depth_shape[0] / im_shape[0] data = [ { "image_path": image, "pose": pose, "depth": cv2.resize(depth.numpy(), target_depth_shape[::-1], interpolation=cv2.INTER_LINEAR), "source_K": K, "image_K": K * image_scale, "depth_K": K * depth_scale, "pts3d": pts, "im_conf": im_conf, "im_shape_target": image_hw, "depth_shape_target": target_depth_shape, "shape_original": im_shape, } for image, pose, depth, K, pts, im_conf in zip(images, poses, depths, Ks, pts3d, im_confs) ] return data def es_wrap(data): try: return export_scene(data) except Exception as e: print(e) return None def export_scene(data): vggt_dataset, arkit_dataset, i, out_dir, processing_args = data vggt_scene = vggt_dataset[i] scene_id = vggt_scene.id arkit_scene = arkit_dataset[scene_id] out_path = out_dir / scene_id out_path.mkdir(parents=True, exist_ok=True) # if os.path.exists(out_path / f'{vggt_scene.id}_vh_clean_2.ply'): # return arkit_scene.frames = arkit_scene.frames[-100:] aligner = build_aligner_1p1d(source_scene=vggt_scene, target_scene=arkit_scene) scene = aligner.align(vggt_scene, inplace=True) # scene = arkit_scene K_color = arkit_scene[0].image_intrinsics K_depth = arkit_scene[0].depth_intrinsics intrinsics_path = out_path / "intrinsic" intrinsics_path.mkdir(parents=True, exist_ok=True) color_path = out_path / "color" color_path.mkdir(parents=True, exist_ok=True) depth_path = out_path / "depth" depth_path.mkdir(parents=True, exist_ok=True) pose_path = out_path / "pose" pose_path.mkdir(parents=True, exist_ok=True) np.savetxt(intrinsics_path / "intrinsic_color.txt", K_color) np.savetxt(intrinsics_path / "intrinsic_depth.txt", K_depth) np.savetxt(intrinsics_path / "extrinsic_color.txt", np.eye(4)) np.savetxt(intrinsics_path / "extrinsic_depth.txt", np.eye(4)) tsdffusion = o3d.pipelines.integration.ScalableTSDFVolume( voxel_length=0.025, sdf_trunc=0.1, color_type=o3d.pipelines.integration.TSDFVolumeColorType.RGB8 ) print("rotation angle", arkit_scene.rotation_angle) for frame in tqdm(scene.frames): # image = rotate_image(frame.image, arkit_scene.rotation_angle) # # image = frame.image # h, w = image.shape[:2] # depth = frame.depth.astype(np.float32) # depth = cv2.resize(depth, (w, h), interpolation=cv2.INTER_LINEAR) # color = o3d.geometry.Image(image) np.savetxt(str(pose_path / f'{frame.frame_id}.txt'), frame.pose) # cv2.imwrite(str(color_path / f'{frame.frame_id}.jpg'), image[..., ::-1]) # cv2.imwrite(str(depth_path / f'{frame.frame_id}.png'), (depth * 1000.).astype(np.uint16)) # fx, fy = K_color[0, 0], K_color[1, 1] # cx, cy = K_color[0, 2], K_color[1, 2] # dh, dw = depth.shape # depth_o3d = o3d.geometry.Image(depth) # rgbd = o3d.geometry.RGBDImage.create_from_color_and_depth( # color, depth_o3d, depth_trunc=10., convert_rgb_to_intensity=False, depth_scale=1.0 # ) # camera_o3d = o3d.camera.PinholeCameraIntrinsic(w, h, fx, fy, cx, cy) # tsdffusion.integrate( # rgbd, camera_o3d, # np.linalg.inv(frame.pose), # ) # pc = tsdffusion.extract_point_cloud() # pc.voxel_down_sample(voxel_size=0.025) # o3d.io.write_point_cloud(str(out_path / f'{scene.id}_vh_clean_2.ply'), pc) vggt_dataset = VGGTDataset("/home/jovyan/users/bulat/workspace/3drec/vggt/output/arkit_new/") arkit_dataset = ARKitDataset("/workspace-SR006.nfs2/datasets/arkitscenes/offline_prepared_data/posed_images/") processing_args = { "voxel_size": 0.025, "out_dir": Path("data/arkit_dust3r_posed/processed"), } val_path = Path("../") / "OKNO/data/arkitscenes/arkitscenes_offline_infos_val.pkl" out_dir = Path("data/arkit_vggt/processed") with open(val_path, "rb") as f: data = pickle.load(f) data_list = data["data_list"] val_scenes = [scene["lidar_points"]["lidar_path"] for scene in data_list] def extract_name(item): return item.split("_")[0] val_scenes = [extract_name(scene) for scene in val_scenes] data = [(vggt_dataset, arkit_dataset, i, out_dir, processing_args) for i in range(len(vggt_dataset))] thread_map(es_wrap, data, chunksize=128) # break