custom_nodes/ComfyUI-SCAIL-Pose/render_3d/render_torch.py

import torch
import numpy as np
import random
import math


def flatten_specs(specs_list):
    """
    Flatten specs_list into numpy arrays + index tables.
    Returns:
        starts: (N, 3) float32
        ends:   (N, 3) float32
        colors: (N, 4) float32
        frame_offset: (num_frames,) int32
        frame_count:  (num_frames,) int32
    """
    starts, ends, colors = [], [], []
    frame_offset, frame_count = [], []
    offset = 0
    for specs in specs_list:
        frame_offset.append(offset)
        frame_count.append(len(specs))
        for s, e, c in specs:
            starts.append(s)
            ends.append(e)
            colors.append(c)
        offset += len(specs)

    # Handle empty case
    if len(starts) == 0:
        return (
            np.zeros((0, 3), dtype=np.float32),
            np.zeros((0, 3), dtype=np.float32),
            np.zeros((0, 4), dtype=np.float32),
            np.array(frame_offset, dtype=np.int32),
            np.array(frame_count, dtype=np.int32),
        )

    return (
        np.array(starts, dtype=np.float32),
        np.array(ends, dtype=np.float32),
        np.array(colors, dtype=np.float32),
        np.array(frame_offset, dtype=np.int32),
        np.array(frame_count, dtype=np.int32),
    )


def render_whole(
    specs_list, H=480, W=640, fx=500, fy=500, cx=240, cy=320, radius=21.5, device=None
):
    """
    Render cylinders using PyTorch ray marching.
    """
    if device is None:
        device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

    starts_np, ends_np, colors_np, frame_offset_np, frame_count_np = flatten_specs(
        specs_list
    )

    # Check if there is anything to render
    if len(starts_np) == 0:
        return [np.zeros((H, W, 4), dtype=np.uint8) for _ in range(len(specs_list))]

    # Move geometry data to device
    all_starts = torch.from_numpy(starts_np).to(device).float()
    all_ends = torch.from_numpy(ends_np).to(device).float()
    all_colors = torch.from_numpy(colors_np).to(device).float()

    # Calculate global z bounds for simple culling/near-far plane setting
    z_min_val = min(starts_np[:, 2].min(), ends_np[:, 2].min())
    z_max_val = max(starts_np[:, 2].max(), ends_np[:, 2].max())

    znear = 0.1
    zfar = max(min(z_max_val, 25000), 10000)

    # Prepare rays for the whole image
    # Grid of coordinates
    y_coords, x_coords = torch.meshgrid(
        torch.arange(H, device=device).float(),
        torch.arange(W, device=device).float(),
        indexing="ij",
    )

    # Camera intrinsics to ray directions
    u = (x_coords - cx) / fx
    v = (y_coords - cy) / fy
    z = torch.ones_like(u)

    # Ray directions in camera/world space (assuming identity rotation for camera)
    ray_dirs = torch.stack([u, v, z], dim=-1)
    ray_dirs = ray_dirs / torch.norm(ray_dirs, dim=-1, keepdim=True)  # (H, W, 3)

    ray_origins = torch.zeros(
        (H, W, 3), device=device
    )  # Camera at (0,0,0) [C variable in taichi]

    light_dir = torch.tensor([0.0, 0.0, 1.0], device=device)

    # Rendering parameters
    MAX_STEPS = 100
    EPSILON = 1e-3

    rendered_frames = []

    # We render frame by frame to avoid OOM with large cylinder counts per frame
    # But batching pixels is implicitly done by operating on full (H, W) tensors.

    for i in range(len(specs_list)):
        start_idx = frame_offset_np[i]
        count = frame_count_np[i]

        if count == 0:
            rendered_frames.append(np.zeros((H, W, 4), dtype=np.uint8))
            continue

        # Get cylinders for this frame
        curr_starts = all_starts[start_idx : start_idx + count]  # (M, 3)
        curr_ends = all_ends[start_idx : start_idx + count]  # (M, 3)
        curr_colors = all_colors[start_idx : start_idx + count]  # (M, 4)

        # --- Ray Marching ---

        # Optimization: Precompute cylinder vectors
        ba = curr_ends - curr_starts  # (M, 3)
        ba_len = torch.sqrt((ba * ba).sum(dim=1))
        ba_norm = ba / ba_len.unsqueeze(1)  # Normalized axis

        # We need to find closest cylinder for each pixel.
        # Since M (num cylinders) is small (~20-100), we can broadcast.
        # But (H*W) is large (480*640 = 307200).
        # (H, W, 1, 3) - (1, 1, M, 3) -> Memory heavey.
        # So we flatten pixels.

        pixels_shape = (H * W,)
        flat_ray_dirs = ray_dirs.view(-1, 3)
        flat_ray_origins = ray_origins.view(-1, 3)

        flat_t = torch.ones(pixels_shape[0], device=device) * znear
        flat_active = torch.ones(pixels_shape[0], dtype=torch.bool, device=device)
        flat_hit = torch.zeros(pixels_shape[0], dtype=torch.bool, device=device)
        flat_hit_color = torch.zeros((pixels_shape[0], 4), device=device)
        flat_hit_pos = torch.zeros(
            (pixels_shape[0], 3), device=device
        )  # Store hit pos for normal calc

        # To avoid OOM, checking 300k pixels vs 100 cylinders is fine (30MB matrices).
        # Let's verify:
        # Points P: (N_pix, 3)
        # Cyl Start A: (N_cyl, 3)
        # P - A: (N_pix, N_cyl, 3). 300k * 100 * 3 * 4bytes ~= 360MB.
        # This fits in standard GPU memory easily.

        depth_near = max(z_min_val, 0.1)
        depth_far = min(z_max_val + 6000, 20000)

        for step in range(MAX_STEPS):
            if not flat_active.any():
                break

            # Current points for active rays
            # Only compute for active rays to save time?
            # Indexing might be slower than just masking. Let's try masking.

            p = flat_ray_origins + flat_ray_dirs * flat_t.unsqueeze(1)  # (N_pix, 3)

            # --- SDF Calculation ---
            # Broadcast p against cylinders
            # We only need to compute SDF for active pixels, but let's do all for simplicity first,
            # or better: filter indices.

            active_indices = torch.nonzero(flat_active).squeeze()
            if active_indices.numel() == 0:
                break

            p_active = p[active_indices]  # (K, 3)

            pa = p_active.unsqueeze(1) - curr_starts.unsqueeze(0)  # (K, M, 3)

            # proj
            # ba_norm: (M, 3) -> (1, M, 3)
            proj = (pa * ba_norm.unsqueeze(0)).sum(dim=-1)  # (K, M)

            # clamp
            proj_clamped = proj.clamp(min=0.0).min(ba_len.unsqueeze(0))  # (K, M)

            # vec to closest point on axis
            closest_on_axis = curr_starts.unsqueeze(0) + proj_clamped.unsqueeze(
                -1
            ) * ba_norm.unsqueeze(0)  # (K, M, 3)

            # dist
            dist_vec = p_active.unsqueeze(1) - closest_on_axis
            dist_euc = torch.norm(dist_vec, dim=-1)  # (K, M)
            sdf = dist_euc - radius  # (K, M)

            # Combine all cylinders (Union = min)
            min_sdf, min_idx = sdf.min(dim=1)  # (K,)

            # Update t
            # If min_sdf < EPSILON, we hit
            # If flat_t > zfar, we miss

            # Map back to full arrays
            current_t_vals = flat_t[active_indices]

            hit_cond = min_sdf < EPSILON
            miss_cond = current_t_vals > zfar

            # For hits
            new_hits = hit_cond & (~miss_cond)
            # Only update hit info for newly hit rays

            # We need to write back results
            # Global indices of new hits
            hit_global_idx = active_indices[new_hits]

            if hit_global_idx.numel() > 0:
                flat_hit[hit_global_idx] = True
                flat_active[hit_global_idx] = False
                flat_hit_pos[hit_global_idx] = p_active[new_hits]  # Store position

                # Get color of closest cylinder
                closest_cyl_idx = min_idx[new_hits]
                flat_hit_color[hit_global_idx] = curr_colors[closest_cyl_idx]

            # For misses
            miss_global_idx = active_indices[miss_cond]
            if miss_global_idx.numel() > 0:
                flat_active[miss_global_idx] = False

            # Step t
            # Only step remaining active
            still_active_local = ~(hit_cond | miss_cond)
            if still_active_local.any():
                step_dist = min_sdf[still_active_local]
                # Avoid stepping too small to prevent stuck
                step_dist = torch.max(step_dist, torch.tensor(1e-4, device=device))

                active_global_idx = active_indices[still_active_local]
                flat_t[active_global_idx] += step_dist

        # --- Shading ---
        # Compute normals for all hit pixels
        hit_indices = torch.nonzero(flat_hit).squeeze()

        if hit_indices.numel() > 0:
            p_hit = flat_hit_pos[hit_indices]  # (NumHits, 3)
            hit_cols = flat_hit_color[hit_indices]  # (NumHits, 4)

            # Finite difference normal
            e = 1e-3

            # We need a function to compute scene SDF at arbitrary points quickly
            def get_sdf_batch(points):
                # points: (N, 3)
                # returns: (N,) min sdf
                # Re-use curr_starts, curr_ends logic

                # Chunking if too large?
                # Assuming it fits since points are subset of image

                pa = points.unsqueeze(1) - curr_starts.unsqueeze(0)  # (N, M, 3)
                proj = (pa * ba_norm.unsqueeze(0)).sum(dim=-1)
                proj_clamped = proj.clamp(min=0.0).min(ba_len.unsqueeze(0))

                closest = curr_starts.unsqueeze(0) + proj_clamped.unsqueeze(
                    -1
                ) * ba_norm.unsqueeze(0)
                dist = torch.norm(points.unsqueeze(1) - closest, dim=-1)
                sdf = dist - radius
                return sdf.min(dim=1)[0]

            def get_normal_batch(points):
                # Central difference
                dx = get_sdf_batch(
                    points + torch.tensor([e, 0, 0], device=device)
                ) - get_sdf_batch(points - torch.tensor([e, 0, 0], device=device))
                dy = get_sdf_batch(
                    points + torch.tensor([0, e, 0], device=device)
                ) - get_sdf_batch(points - torch.tensor([0, e, 0], device=device))
                dz = get_sdf_batch(
                    points + torch.tensor([0, 0, e], device=device)
                ) - get_sdf_batch(points - torch.tensor([0, 0, e], device=device))
                n = torch.stack([dx, dy, dz], dim=-1)
                return n / (torch.norm(n, dim=-1, keepdim=True) + 1e-8)

            normals = get_normal_batch(p_hit)

            # Blinn-Phong
            # View dir is -ray_dir
            view_dir = -flat_ray_dirs[hit_indices]
            view_dir = view_dir / torch.norm(view_dir, dim=-1, keepdim=True)

            # Light dir (0,0,1)
            # Diffuse
            # max(n.dot(-light_dir), 0) -> note taichi code used -light_dir for diffuse?
            # Taichi: diff = max(n.dot(-light_dir), 0.0) where light_dir = [0,0,1]
            # So light comes from +Z (camera).

            diff = torch.clamp(
                (normals * (-light_dir)).sum(dim=-1), min=0.0
            )  # (NumHits,)

            # Specular
            half_dir = (view_dir + (-light_dir)).float()
            half_dir = half_dir / (torch.norm(half_dir, dim=-1, keepdim=True) + 1e-8)

            spec = torch.clamp((normals * half_dir).sum(dim=-1), min=0.0)
            spec = spec**32

            # Depth factor
            z_vals = p_hit[:, 2]
            depth_factor = 1.0 - (z_vals - depth_near) / (depth_far - znear)
            depth_factor = depth_factor.clamp(0.0, 1.0)

            # Combine
            diffuse_term = 0.3 + 0.7 * diff
            base_rgb = (
                hit_cols[:, :3]
                * diffuse_term.unsqueeze(-1)
                * depth_factor.unsqueeze(-1)
            )

            highlight = (
                torch.tensor([1.0, 1.0, 1.0], device=device)
                * (0.5 * spec.unsqueeze(-1))
                * depth_factor.unsqueeze(-1)
            )

            final_rgb = base_rgb + highlight

            # Assign back
            flat_hit_color[hit_indices, :3] = final_rgb
            flat_hit_color[hit_indices, 3] = hit_cols[:, 3]  # Alpha

        # Reshape to image
        frame_img = flat_hit_color.view(H, W, 4)

        # Convert to numpy uint8
        frame_np = (frame_img.clamp(0, 1) * 255).byte().cpu().numpy()
        rendered_frames.append(frame_np)

    return rendered_frames


def random_cylinder():
    """Generate a random cylinder (start, end, color)."""
    # Start point [-200,200]^2, z in [300,400]
    ax = random.uniform(-200, 200)
    ay = random.uniform(-200, 200)
    az = random.uniform(300, 400)
    start = [ax, ay, az]

    # Random direction and length
    theta = random.uniform(0, 2 * math.pi)
    phi = random.uniform(-math.pi / 4, math.pi / 4)  # Tilt angle
    L = 100
    dx = math.cos(phi) * math.cos(theta)
    dy = math.cos(phi) * math.sin(theta)
    dz = math.sin(phi)
    end = [ax + dx * L, ay + dy * L, az + dz * L]

    # Random color (RGB + alpha=1)
    color = [random.random(), random.random(), random.random(), 1.0]

    return (start, end, color)


def generate_specs_list(num_frames=120, min_cyl=10, max_cyl=120):
    """Generate specs_list, each frame has several random cylinders."""
    specs_list = []
    for _ in range(num_frames):
        n_cyl = random.randint(min_cyl, max_cyl)
        specs = [random_cylinder() for _ in range(n_cyl)]
        specs_x_shift = [
            (
                [spec[0][0] + 50, spec[0][1], spec[0][2]],
                [spec[1][0] + 50, spec[1][1], spec[1][2]],
                spec[2],
            )
            for spec in specs
        ]
        specs_list.append(specs)
        specs_list.append(specs_x_shift)
    return specs_list