utils/data_utils.py

# Copyright (c) 2025 Hanwen Jiang, Xuweiyi Chen. Adapted for WildRayZer from the RayZer project.

import random
import numpy as np
import torch
import torch.nn as nn
from easydict import EasyDict as edict
from einops import rearrange
import imageio
import math


def create_video_from_frames(frames, output_video_file, framerate=30):
    """
    Creates a video from a sequence of frames.

    Parameters:
        frames (numpy.ndarray): Array of image frames (shape: N x H x W x C).
        output_video_file (str): Path to save the output video file.
        framerate (int, optional): Frames per second for the video. Default is 30.
    """
    frames = np.asarray(frames)

    # Normalize frames if values are in [0,1] range
    if frames.max() <= 1:
        frames = (frames * 255).astype(np.uint8)

    imageio.mimsave(output_video_file, frames, fps=framerate, quality=8)


# used in lvsm repo, which is slightly different from rayzer's view sampling setting
class ProcessData(nn.Module):
    def __init__(self, config):
        super().__init__()
        self.config = config

    @torch.no_grad()
    def compute_rays(self, c2w, fxfycxcy, h=None, w=None, device="cuda"):
        """
        Args:
            c2w (torch.tensor): [b, v, 4, 4]
            fxfycxcy (torch.tensor): [b, v, 4]
            h (int): height of the image
            w (int): width of the image
        Returns:
            ray_o (torch.tensor): [b, v, 3, h, w]
            ray_d (torch.tensor): [b, v, 3, h, w]
        """

        b, v = c2w.size()[:2]
        c2w = c2w.reshape(b * v, 4, 4)

        fx, fy, cx, cy = fxfycxcy[:, :, 0], fxfycxcy[:, :, 1], fxfycxcy[:, :, 2], fxfycxcy[:, :, 3]
        h_orig = int(2 * cy.max().item())  # Original height (estimated from the intrinsic matrix)
        w_orig = int(2 * cx.max().item())  # Original width (estimated from the intrinsic matrix)
        if h is None or w is None:
            h, w = h_orig, w_orig

        # in case the ray/image map has different resolution than the original image
        if h_orig != h or w_orig != w:
            fx = fx * w / w_orig
            fy = fy * h / h_orig
            cx = cx * w / w_orig
            cy = cy * h / h_orig

        fxfycxcy = fxfycxcy.reshape(b * v, 4)
        y, x = torch.meshgrid(torch.arange(h), torch.arange(w), indexing="ij")
        y, x = y.to(device), x.to(device)
        x = x[None, :, :].expand(b * v, -1, -1).reshape(b * v, -1)
        y = y[None, :, :].expand(b * v, -1, -1).reshape(b * v, -1)
        x = (x + 0.5 - fxfycxcy[:, 2:3]) / fxfycxcy[:, 0:1]
        y = (y + 0.5 - fxfycxcy[:, 3:4]) / fxfycxcy[:, 1:2]
        z = torch.ones_like(x)
        ray_d = torch.stack([x, y, z], dim=2)  # [b*v, h*w, 3]
        ray_d = torch.bmm(ray_d, c2w[:, :3, :3].transpose(1, 2))  # [b*v, h*w, 3]
        ray_d = ray_d / torch.norm(ray_d, dim=2, keepdim=True)  # [b*v, h*w, 3]
        ray_o = c2w[:, :3, 3][:, None, :].expand_as(ray_d)  # [b*v, h*w, 3]

        ray_o = rearrange(ray_o, "(b v) (h w) c -> b v c h w", b=b, v=v, h=h, w=w, c=3)
        ray_d = rearrange(ray_d, "(b v) (h w) c -> b v c h w", b=b, v=v, h=h, w=w, c=3)

        return ray_o, ray_d

    def fetch_views(self, data_batch, has_target_image=False, target_has_input=True):
        """
        Splits the input data batch into input and target sets.

        Args:
            data_batch (dict): Contains input tensors with the following keys:
                - 'image' (torch.Tensor): Shape [b, v, c, h, w], optional for some target views
                - 'fxfycxcy' (torch.Tensor): Shape [b, v, 4]
                - 'c2w' (torch.Tensor): Shape [b, v, 4, 4]
            target_has_input (bool): If True, target includes input views.

        Returns:
            tuple: (input_dict, target_dict), both as EasyDict objects.

        """
        # randomize input views if dynamic_input_view_num is True and not in inference mode
        if self.config.training.get("dynamic_input_view_num", False) and (
            not self.config.inference.get("if_inference", False)
        ):
            self.config.training.num_input_views = np.random.randint(2, 5)

        input_dict, target_dict = {}, {}
        # index = [] save for future use if we want to select specific views
        # Handle different data formats from adapters
        if "input" in data_batch and "target" in data_batch:
            # Handle nested format from simple_stereo4d_adapter
            # The DataLoader batches the data, so we get [B, V, C, H, W] format
            input_images = data_batch["input"]["image"]  # [B, num_input, 3, H, W]
            target_images = data_batch["target"]["image"]  # [B, num_target, 3, H, W]

            # Concatenate along the view dimension (dim=1)
            all_images = torch.cat([input_images, target_images], dim=1)  # [B, V, 3, H, W]

            # Create flat structure that rest of fetch_views expects
            data_batch_flat = {
                "image": all_images,  # [B, V, 3, H, W] - already has batch dim
                "scene_name": data_batch.get("scene_name", ""),
                "fps": data_batch.get("fps", 0.0),
                "frame_count": data_batch.get("frame_count", 0),
            }

            # Add time if available
            if "time" in data_batch["input"] and "time" in data_batch["target"]:
                input_time = data_batch["input"]["time"]  # [B, num_input, 1]
                target_time = data_batch["target"]["time"]  # [B, num_target, 1]
                all_time = torch.cat([input_time, target_time], dim=1)  # [B, V, 1]
                data_batch_flat["time"] = all_time  # [B, V, 1]

            # Add index field for visualization (required by metric_utils.py)
            # Create index from time information and batch indices
            B = all_images.shape[0]
            V = all_images.shape[1]

            # Use time values to create pseudo frame indices
            # Convert normalized time [-1, 1] to frame-like indices [0, 1000]
            if "time" in data_batch_flat:
                time_tensor = data_batch_flat["time"]  # [B, V, 1]
                pseudo_frame_indices = (
                    (time_tensor.squeeze(-1) + 1.0) * 500
                ).long()  # [B, V] in [0, 1000]
            else:
                # Fallback: use view indices as frame indices
                pseudo_frame_indices = (
                    torch.arange(V, device=all_images.device).unsqueeze(0).expand(B, -1)
                )  # [B, V]

            # Create scene indices (batch indices repeated for each view)
            scene_indices = (
                torch.arange(B, device=all_images.device).unsqueeze(1).expand(B, V)
            )  # [B, V]

            # Combine frame and scene indices
            index_field = torch.stack([pseudo_frame_indices, scene_indices], dim=-1)  # [B, V, 2]
            data_batch_flat["index"] = index_field

            data_batch = data_batch_flat
            num_target_views, num_views, bs = (
                self.config.training.num_target_views,
                data_batch["image"].size(1),
                data_batch["image"].size(0),
            )
        elif "c2w" in data_batch:
            num_target_views, num_views, bs = (
                self.config.training.num_target_views,
                data_batch["c2w"].size(1),
                data_batch["image"].size(0),
            )
        else:
            # For pose-free datasets, get dimensions from image tensor
            num_target_views, num_views, bs = (
                self.config.training.num_target_views,
                data_batch["image"].size(1),
                data_batch["image"].size(0),
            )
        assert (
            num_target_views < num_views
        ), f"We have {num_views} views, but we want to select {num_target_views} target views. This is more than the total number of views we have."

        K = int(self.config.training.num_input_views)
        T = int(num_target_views)
        # Decide the target view indices
        if target_has_input:
            # Target views are the remaining views after inputs (maintaining temporal order)
            # Since your dataset puts inputs first, targets are views K to V-1
            target_indices = list(range(K, num_views))  # Views after the first K input views
            index = torch.tensor(
                [
                    target_indices[
                        :num_target_views
                    ]  # Take the first num_target_views from remaining
                    for _ in range(bs)
                ],
                dtype=torch.long,
                device=data_batch["image"].device,
            )  # [b, num_target_views]
        else:
            # assert (
            #     self.config.training.num_input_views + num_target_views <= self.config.training.num_views
            # ), f"We have {self.config.training.num_views} views in total, but we want to select {self.config.training.num_input_views} input views and {num_target_views} target views. This is more than the total number of views we have."

            # index = torch.tensor([
            #     [self.config.training.num_views - 1 - j for j in range(num_target_views)]
            #     for _ in range(bs)
            # ], dtype=torch.long, device=data_batch["image"].device)
            # index = torch.sort(index, dim=1).values # [b, num_target_views]
            assert (
                K + T <= num_views
            ), f"Need K (inputs) + T (targets) <= num_views, got {K}+{T}>{num_views}"
            base = torch.arange(K, K + T, device=data_batch["image"].device)  # [T]
            index = base.unsqueeze(0).expand(bs, -1).to(torch.long)  # [b, T]

        skip_keys = {
            "scene_name",
            "fps",
            "frame_count",
            "context_indices",
            "target_indices",
            "context_source_indices",
            "target_source_indices",
            "context_original_filenames",
            "target_original_filenames",
            "context_gt_original_filenames",
            "target_gt_original_filenames",
        }

        for key, value in data_batch.items():
            if key in skip_keys:
                # Preserve metadata tensors/lists without slicing by view dimension
                input_dict[key] = value
                target_dict[key] = value
                continue

            # Handle tensors with view dimensions
            if isinstance(value, torch.Tensor) and len(value.shape) >= 2:
                input_dict[key] = value[:, : self.config.training.num_input_views, ...]
            else:
                # Handle other data types or unexpected shapes
                input_dict[key] = value
                target_dict[key] = value
                continue

            to_expand_dim = value.shape[2:]  # [b, v, ...] -> [...]
            expanded_index = index.view(
                index.shape[0], index.shape[1], *(1,) * len(to_expand_dim)
            ).expand(-1, -1, *to_expand_dim)

            # Don't have target image supervision
            if key == "image" and not has_target_image:
                continue
            else:
                target_dict[key] = torch.gather(value, dim=1, index=expanded_index)

        height, width = data_batch["image"].shape[3], data_batch["image"].shape[4]
        input_dict["image_h_w"] = (height, width)
        target_dict["image_h_w"] = (height, width)

        input_dict, target_dict = edict(input_dict), edict(target_dict)

        return input_dict, target_dict

    @torch.no_grad()
    def forward(self, data_batch, has_target_image=True, target_has_input=True, compute_rays=True):
        """
        Preprocesses the input data batch and (optionally) computes ray_o and ray_d.

        Args:
            data_batch (dict): Contains input tensors with the following keys:
                - 'image' (torch.Tensor): Shape [b, v, c, h, w]
                - 'fxfycxcy' (torch.Tensor): Shape [b, v, 4]
                - 'c2w' (torch.Tensor): Shape [b, v, 4, 4]
            has_target_image (bool): If True, target views have image supervision.
            target_has_input (bool): If True, target views can be sampled from input views.
            compute_rays (bool): If True, compute ray_o and ray_d.

        Returns:
            Input and Target data_batch (dict): Contains processed tensors with the following keys:
                - 'image' (torch.Tensor): Shape [b, v, c, h, w]
                - 'fxfycxcy' (torch.Tensor): Shape [b, v, 4]
                - 'c2w' (torch.Tensor): Shape [b, v, 4, 4]
                - 'ray_o' (torch.Tensor): Shape [b, v, 3, h, w]
                - 'ray_d' (torch.Tensor): Shape [b, v, 3, h, w]
                - 'image_h_w' (tuple): (height, width)
        """
        input_dict, target_dict = self.fetch_views(
            data_batch, has_target_image=has_target_image, target_has_input=target_has_input
        )

        if compute_rays:
            for dict in [input_dict, target_dict]:
                c2w = dict["c2w"]
                fxfycxcy = dict["fxfycxcy"]
                image_height, image_width = dict["image_h_w"]

                ray_o, ray_d = self.compute_rays(
                    c2w, fxfycxcy, image_height, image_width, device=data_batch["image"].device
                )
                dict["ray_o"], dict["ray_d"] = ray_o, ray_d

        return input_dict, target_dict


class SplitData(nn.Module):
    def __init__(self, config):
        super().__init__()
        self.config = config

        # Basic check: we want num_input_views + num_target_views = num_views
        assert (
            self.config.training.num_views
            == self.config.training.num_input_views + self.config.training.num_target_views
        ), "num_input_views + num_target_views must equal num_views"

        # Precompute input and target indices (no overlap, evenly spaced)
        self.input_pattern, self.target_pattern = self._build_indices(
            total_views=self.config.training.num_views,
            num_input_views=self.config.training.num_input_views,
            num_target_views=self.config.training.num_target_views,
        )

        print(
            "When not using random index, input and target indices are:",
            self.input_pattern,
            self.target_pattern,
        )
        # tmp1, tmp2 = self.input_pattern[-1].clone(), self.target_pattern[-1].clone()
        # self.target_pattern[-1] = tmp1
        # self.input_pattern[-1] = tmp2

    @torch.no_grad()
    def forward(self, data_batch, random_index=True):
        """
        Each tensor in data_batch has shape [B, V, ...].
        We'll slice along dimension 1 (the 'view' dimension).
        """
        input_dict, target_dict = {}, {}
        B, V = data_batch["image"].shape[:2]
        batch_idx = torch.arange(B).unsqueeze(1).to(data_batch["image"].device)

        # Check if using dataset-provided indices (evaluation mode)
        use_dataset_indices = "context_indices" in data_batch and "target_indices" in data_batch

        if use_dataset_indices:
            # use loaded view indices, for evaluation
            input_pattern = data_batch["context_indices"]
            target_pattern = data_batch["target_indices"]
        else:
            # for training
            if random_index:
                input_pattern, target_pattern = self.get_random_index(B, V)
            else:
                input_pattern, target_pattern = self.input_pattern.unsqueeze(0).repeat(
                    B, 1
                ), self.target_pattern.unsqueeze(0).repeat(B, 1)

        for key, value in data_batch.items():
            if key in set(
                [
                    "scene_name",
                    "context_indices",
                    "target_indices",
                    "dataset_sources",
                    "dataset_source",
                ]
            ):
                continue
            # value shape: [B, V, ...]
            B, V = value.shape[:2]
            # Only validate view count for training (not for evaluation with dataset-provided indices)
            if not use_dataset_indices:
                expected_views = self.config.training.num_views
                if V != expected_views:
                    raise ValueError(f"Expected {key} to have {expected_views} views, got {V}.")

            input_dict[key] = value[batch_idx, input_pattern, ...]
            target_dict[key] = value[batch_idx, target_pattern, ...]

        # Add scene_name to both dicts (needed for evaluation)
        # scene_name should be indexed by batch when it's a list
        if "scene_name" in data_batch:
            scene_names = data_batch["scene_name"]
            # If it's a list (batched), keep as list; if single string, keep as string
            input_dict["scene_name"] = scene_names
            target_dict["scene_name"] = scene_names

        return edict(input_dict), edict(target_dict), input_pattern, target_pattern

    def _build_indices(self, total_views, num_input_views, num_target_views):
        """
        Build two arrays of indices for input and target such that
        they don't overlap and cover all views evenly.
        E.g. total_views=24, num_input_views=16, num_target_views=8
        => input might be [0,1,3,4,6,7,9,10,...], target [2,5,8,11,...]
        """
        # Simple approach: gcd-based grouping
        g = math.gcd(num_input_views, num_target_views)
        group_size = total_views // g  # number of consecutive indices per group
        in_per_group = num_input_views // g
        tar_per_group = num_target_views // g

        input_indices = []
        target_indices = []

        for group_idx in range(g):
            start = group_idx * group_size
            block = list(range(start, start + group_size))
            # first part goes to inputs
            input_indices.extend(block[:in_per_group])
            # next part goes to targets
            target_indices.extend(block[in_per_group : in_per_group + tar_per_group])

        # Convert to torch.LongTensor
        input_indices = torch.tensor(input_indices, dtype=torch.long)
        target_indices = torch.tensor(target_indices, dtype=torch.long)
        input_indices, _ = torch.sort(input_indices)
        target_indices, _ = torch.sort(target_indices)

        return input_indices, target_indices

    def get_random_index(self, b, v):
        total_views = self.config.training.num_views
        num_input_views = self.config.training.num_input_views
        num_target_views = self.config.training.num_target_views
        random_shuffle = self.config.training.view_selector.get("shuffle", False)

        assert (
            num_input_views + num_target_views == total_views
        ), "Mismatch in total views allocation."

        rand_vals = torch.rand(b, v)  # shape [B, V]
        perms = rand_vals.argsort(dim=1)  # shape [B, V]

        # Ensure at least one index in input is smaller than all in target, and one index in input is larger than all in target
        idx_part1 = torch.zeros((b, num_input_views), dtype=torch.long, device=perms.device)
        idx_part2 = torch.zeros((b, num_target_views), dtype=torch.long, device=perms.device)

        for i in range(b):
            # Ensure the first index in input is always 0 and the last index is always v-1
            idx_part1[i, 0] = 0
            idx_part1[i, -1] = v - 1

            # Remaining indices to choose from
            remaining_indices = torch.arange(1, v - 1, device=perms.device)  # Exclude 0 and v-1

            # Randomly sample (num_input_views - 2) indices from remaining
            middle_size = num_input_views - 2
            middle_indices = remaining_indices[torch.randperm(len(remaining_indices))[:middle_size]]
            middle_indices, _ = middle_indices.sort()  # Ensure sorted order
            # Assign middle indices to idx_part1
            idx_part1[i, 1:-1] = middle_indices

            # Target indices are the remaining ones
            idx_part2_indeices = torch.tensor(
                [x for x in range(v) if x not in idx_part1[i]], device=perms.device
            )
            idx_part2_indeices, _ = idx_part2_indeices.sort()  # Ensure sorted order for target
            idx_part2[i] = idx_part2_indeices

            if random_shuffle:
                idx_part1[i] = idx_part1[i][torch.randperm(num_input_views)]
                idx_part2[i] = idx_part2[i][torch.randperm(num_target_views)]

        return idx_part1, idx_part2