"""OmniWorld loader. On-disk layout (per scene, under ``data_root``):: / color/.png RGB frames camera/split_.json focals / cx / cy droidclib/split_.json DROID-SLAM extrinsics (T, 4, 4) text/_.json caption windows An offline index (``scripts/build_omniworld_index.py``) lists one 81-frame training window per entry. DROID-SLAM emits camera-from-world extrinsics, so we invert them to camera-to-world. Intrinsics are given for the original resolution and rescaled to the training width when computing FOV. """ from __future__ import annotations import json import math from pathlib import Path from typing import Any import numpy as np from PIL import Image from scope.data.common import BaseCameraVideoDataset class OmniWorldDataset(BaseCameraVideoDataset): def __init__( self, data_root: str, index_path: str, num_frames: int = 81, height: int = 480, width: int = 832, pose_is_w2c: bool = True, caption_fields: tuple[str, ...] = ( "Video_Caption", "Short_Caption", "Background_Caption", ), max_videos: int | None = None, near_depth_map: dict[str, float] | None = None, trajectory_scale: float = 1.0, return_first_frame: bool = True, ) -> None: super().__init__( num_frames=num_frames, height=height, width=width, near_depth_map=near_depth_map, trajectory_scale=trajectory_scale, return_first_frame=return_first_frame, ) self.data_root = Path(data_root) self.pose_is_w2c = pose_is_w2c self.caption_fields = caption_fields index_file = Path(index_path) if not index_file.is_file(): raise FileNotFoundError( f"OmniWorld index not found: {index_file}. Build it with " f"scripts/build_omniworld_index.py." ) entries = json.loads(index_file.read_text(encoding="utf-8")) keep = set(self.filter_ids_by_near_depth([self._entry_id(e) for e in entries])) entries = [e for e in entries if self._entry_id(e) in keep] if max_videos is not None: entries = entries[:max_videos] self.entries = entries if not self.entries: raise ValueError(f"No OmniWorld windows found for {index_file}") print(f"[OmniWorld] {len(self.entries)} windows") @staticmethod def _entry_id(entry: dict[str, Any]) -> str: return f"{entry['scene']}_split{int(entry['split_idx'])}_{int(entry['frame_start']):06d}" def __len__(self) -> int: return len(self.entries) def _select_caption(self, captions: dict[str, Any]) -> str: for field in self.caption_fields: value = captions.get(field) if isinstance(value, str) and value.strip(): return value.strip() parts = [v.strip() for v in captions.values() if isinstance(v, str) and v.strip()] return " ".join(parts) def _compute_x_fov(self, scene: str, split_idx: int) -> float: droid = json.loads( (self.data_root / scene / "droidclib" / f"split_{split_idx}.json").read_text() ) intr = droid.get("orig_intrinsic") or droid.get("crop_intrinsic") if intr is not None: fx_orig, w_orig = float(intr["fx"]), float(intr["cx"]) * 2.0 else: camera = json.loads( (self.data_root / scene / "camera" / f"split_{split_idx}.json").read_text() ) focals = camera.get("focals", []) if not focals: return math.radians(60.0) fx_orig, w_orig = float(np.mean(focals)), float(camera.get("cx", 640.0)) * 2.0 fx_train = fx_orig * (self.width / max(w_orig, 1.0)) return 2.0 * math.atan(self.width / (2.0 * fx_train)) def _load_raw(self, index: int) -> dict[str, Any]: entry = self.entries[index] scene = entry["scene"] split_idx = int(entry["split_idx"]) frame_start = int(entry["frame_start"]) local_start = int(entry["split_local_start"]) color_dir = self.data_root / scene / "color" pil_frames = [ Image.open(color_dir / f"{i:06d}.png").convert("RGB") for i in range(frame_start, frame_start + self.num_frames) ] droid = json.loads( (self.data_root / scene / "droidclib" / f"split_{split_idx}.json").read_text() ) extrinsics = np.asarray(droid["extrinsics"], dtype=np.float32) window = extrinsics[local_start : local_start + self.num_frames] poses = np.linalg.inv(window) if self.pose_is_w2c else window caption_obj = json.loads( (self.data_root / scene / entry["caption_file"]).read_text(encoding="utf-8") ) caption = self._select_caption(caption_obj.get("captions", caption_obj)) return { "video_id": self._entry_id(entry), "frames": pil_frames, "poses": poses[:, :3], "x_fov": self._compute_x_fov(scene, split_idx), "xi": 0.0, "caption": caption, }