infer.py

"""

Batch VDPM Inference (CLI version of Gradio Demo)



This script replicates the exact logic of gradio_demo.py but for command-line usage.

It supports processing a folder of video files (treated as synchronized multi-view input)

or a single video file.



Usage:

    python vdpm/infer.py --input path/to/videos_folder --output output/

    python vdpm/infer.py --input path/to/video.mp4 --output output/

"""

import os
import sys
import glob
import json
import argparse
import time
import shutil
import gc
from pathlib import Path
from datetime import datetime

import cv2
import numpy as np
import torch
from hydra import compose, initialize
from hydra.core.global_hydra import GlobalHydra

# Set fragmentation fix BEFORE importing torch (or immediately after imports)
os.environ["PYTORCH_CUDA_ALLOC_CONF"] = "expandable_segments:True"

# Add parent dir to path to allow importing dpm/vggt modules if running from root
sys.path.insert(0, str(Path(__file__).parent))

from dpm.model import VDPM
from vggt.utils.load_fn import load_and_preprocess_images
from util.depth import write_depth_to_png

# ============================================================================
# CONFIGURATION (MATCHING GRADIO_DEMO.PY)
# ============================================================================

VIDEO_SAMPLE_HZ = 1.0
USE_HALF_PRECISION = True
USE_QUANTIZATION = False

device = "cuda" if torch.cuda.is_available() else "cpu"

MAX_FRAMES = 5
if device == "cuda":
    torch.backends.cuda.matmul.allow_tf32 = True
    torch.backends.cudnn.allow_tf32 = True
    
    # Check VRAM to auto-scale MAX_FRAMES
    vram_bytes = torch.cuda.get_device_properties(0).total_memory
    vram_gb = vram_bytes / (1024**3)
    
    print(f"✓ GPU Detected: {torch.cuda.get_device_name(0)} ({vram_gb:.1f} GB VRAM)")
    
    if vram_gb >= 22:  # A10G (24GB), A100 (40/80GB), RTX 3090/4090 (24GB)
        MAX_FRAMES = 80
        print(f"   -> High VRAM detected! Increased MAX_FRAMES to {MAX_FRAMES}")
    elif vram_gb >= 14:  # T4 (16GB), 4080 (16GB)
        MAX_FRAMES = 16
        print(f"   -> Medium VRAM detected! Increased MAX_FRAMES to {MAX_FRAMES}")
    elif vram_gb >= 7.5:  # RTX 3070 Ti, 2080, etc (8GB)
        MAX_FRAMES = 8
        print(f"   -> 8GB VRAM detected. Set MAX_FRAMES to {MAX_FRAMES}")
    else:
        MAX_FRAMES = 5
        print(f"   -> Low VRAM (<8GB). Keeping MAX_FRAMES at {MAX_FRAMES} to prevent OOM")

def require_cuda():
    if device != "cuda":
        raise ValueError("CUDA is not available. Check your environment.")


def decode_poses(pose_enc: np.ndarray, image_hw: tuple) -> tuple:
    """Decode VGGT pose encodings to camera matrices."""
    try:
        from vggt.utils.pose_enc import pose_encoding_to_extri_intri
        
        pose_enc_t = torch.from_numpy(pose_enc).float()
        extrinsic, intrinsic = pose_encoding_to_extri_intri(pose_enc_t, image_hw)
        
        extrinsic = extrinsic[0].numpy()  # (N, 3, 4)
        intrinsic = intrinsic[0].numpy()  # (N, 3, 3)
        
        N = extrinsic.shape[0]
        bottom = np.array([0, 0, 0, 1], dtype=np.float32).reshape(1, 1, 4)
        bottom = np.tile(bottom, (N, 1, 1))
        extrinsics_4x4 = np.concatenate([extrinsic, bottom], axis=1)
        
        return extrinsics_4x4, intrinsic
        
    except ImportError:
        print("Warning: vggt not available. Using identity poses.")
        N = pose_enc.shape[1]
        extrinsics = np.tile(np.eye(4, dtype=np.float32), (N, 1, 1))
        
        H, W = image_hw
        fx = fy = max(H, W)
        cx, cy = W / 2, H / 2
        intrinsic = np.array([[fx, 0, cx], [0, fy, cy], [0, 0, 1]], dtype=np.float32)
        intrinsics = np.tile(intrinsic, (N, 1, 1))
        
        return extrinsics, intrinsics


def compute_depths(world_points: np.ndarray, extrinsics: np.ndarray, num_views: int) -> np.ndarray:
    """

    Compute depth maps from world points and camera extrinsics.

    

    Args:

        world_points: (T, V, H, W, 3) world-space 3D points

        extrinsics: (N, 4, 4) camera extrinsics (world-to-camera)

        num_views: Number of camera views

    

    Returns:

        depths: (T, V, H, W) depth maps (Z in camera coordinates)

    """
    T, V, H, W, _ = world_points.shape
    depths = np.zeros((T, V, H, W), dtype=np.float32)
    
    for t in range(T):
        for v in range(V):
            # Get camera extrinsic for this view at this timestep
            # Images are interleaved: [v0_t0, v1_t0, ..., v0_t1, v1_t1, ...]
            img_idx = t * num_views + v
            if img_idx >= len(extrinsics):
                img_idx = v  # Fallback to first timestep's cameras
            
            w2c = extrinsics[img_idx]  # (4, 4)
            R = w2c[:3, :3]  # (3, 3)
            t_vec = w2c[:3, 3]  # (3,)
            
            # Transform world points to camera coordinates
            pts_world = world_points[t, v].reshape(-1, 3)  # (H*W, 3)
            pts_cam = (R @ pts_world.T).T + t_vec  # (H*W, 3)
            
            # Depth is Z in camera coordinates (positive = in front of camera)
            depth = pts_cam[:, 2].reshape(H, W)
            depths[t, v] = depth
    
    return depths

def load_cfg_from_cli() -> "omegaconf.DictConfig":
    if GlobalHydra.instance().is_initialized():
        GlobalHydra.instance().clear()
    # We don't pass overrides from argv here as they interfere with argparse
    with initialize(config_path="configs"):
        return compose(config_name="visualise")

def load_model(cfg) -> VDPM:
    model = VDPM(cfg).to(device)

    # Use a persistent cache directory that Spaces preserves
    cache_dir = os.path.expanduser("~/.cache/vdpm")
    os.makedirs(cache_dir, exist_ok=True)
    model_path = os.path.join(cache_dir, "vdpm_model.pt")
    
    _URL = "https://huggingface.co/edgarsucar/vdpm/resolve/main/model.pt"
    
    # Download only if not cached
    if not os.path.exists(model_path):
        print(f"Downloading model to {model_path}...")
        sd = torch.hub.load_state_dict_from_url(
            _URL,
            file_name="vdpm_model.pt",
            progress=True,
            map_location=device
        )
        torch.save(sd, model_path)
        print(f"✓ Model cached at {model_path}")
    else:
        print(f"✓ Loading cached model from {model_path}")
        sd = torch.load(model_path, map_location=device)
    
    print(model.load_state_dict(sd, strict=True))

    model.eval()
    
    if USE_HALF_PRECISION and not USE_QUANTIZATION:
        if torch.cuda.is_available() and torch.cuda.get_device_capability()[0] >= 8:
            print("Converting model to BF16 precision...")
            model = model.to(torch.bfloat16)
        else:
            print("Converting model to FP16 precision...")
            model = model.half()
    
    if USE_QUANTIZATION:
        try:
            print("Applying INT8 dynamic quantization...")
            model = model.cpu()
            model = torch.quantization.quantize_dynamic(
                model,
                {torch.nn.Linear, torch.nn.Conv2d},
                dtype=torch.qint8
            )
            model = model.to(device)
        except Exception as e:
            print(f"⚠️  Quantization failed: {e}")
            model = model.to(device)
    
    if not USE_QUANTIZATION:
        try:
            print("Compiling model with torch.compile...")
            model = torch.compile(model, mode="reduce-overhead")
        except Exception as e:
            print(f"Warning: torch.compile failed: {e}")
    
    return model

# ============================================================================
# PROCESSING LOGIC (EXACT MATCH TO GRADIO_DEMO)
# ============================================================================

def process_videos_interleaved(input_video_list, target_dir_images):
    """

    Extract frames from multiple videos in a synchronized/interleaved manner.

    Matches handle_uploads logic from gradio_demo.py.

    """
    frame_num = 0
    image_paths = []
    
    # 1. Open all videos
    captures = []
    capture_meta = []
    
    for idx, video_path in enumerate(input_video_list):
        print(f"Preparing video {idx+1}/{len(input_video_list)}: {video_path}")
        
        vs = cv2.VideoCapture(video_path)
        fps = float(vs.get(cv2.CAP_PROP_FPS) or 0.0)
        if fps <= 0: fps = 30.0 # Fallback
        
        frame_interval = max(int(fps / max(VIDEO_SAMPLE_HZ, 1e-6)), 1)
        captures.append(vs)
        capture_meta.append({"interval": frame_interval, "name": video_path})
        
    # 2. Step through them together
    print("Processing videos in interleaved mode...")
    step_count = 0
    active_videos = True
    
    while active_videos:
        active_videos = False
        for i, vs in enumerate(captures):
            if not vs.isOpened():
                continue
            
            gotit, frame = vs.read()
            if gotit:
                active_videos = True # Keep going as long as at least one video has frames
                
                if step_count % capture_meta[i]["interval"] == 0:
                    out_path = os.path.join(target_dir_images, f"{frame_num:06}.png")
                    cv2.imwrite(out_path, frame)
                    image_paths.append(out_path)
                    frame_num += 1
            else:
                vs.release()
        
        step_count += 1
    
    return image_paths


def run_model(target_dir: str, model: VDPM, frame_id_arg=0) -> dict:
    require_cuda()

    image_names = sorted(glob.glob(os.path.join(target_dir, "images", "*")))
    if not image_names:
        raise ValueError("No images found in target_dir.")

    # Load metadata for Multi-View sync
    meta_path = os.path.join(target_dir, "meta.json")
    num_views = 1
    if os.path.exists(meta_path):
        try:
            with open(meta_path, 'r') as f:
                num_views = json.load(f).get("num_views", 1)
        except:
            pass

    # Limit frames to prevent OOM on 8GB GPUs
    if len(image_names) > MAX_FRAMES:
        limit = (MAX_FRAMES // num_views) * num_views
        if limit == 0:
            limit = num_views 
            print(f"⚠️  Warning: MAX_FRAMES={MAX_FRAMES} is smaller than num_views={num_views}. Processing 1 full timestep anyway.")
        
        print(f"⚠️  Limiting to {limit} frames ({limit // num_views} timesteps * {num_views} views) to fit in GPU memory")
        image_names = image_names[:limit]

    print(f"Loading {len(image_names)} images...")
    images = load_and_preprocess_images(image_names).to(device)

    if device == "cuda":
        print(f"GPU memory before inference: {torch.cuda.memory_allocated() / 1024**3:.2f} GB")
    
    print(f"Running inference on {len(image_names)} images ({num_views} synchronized views)...")
    
    # Construct 'views' dictionaries with correct time and camera indices
    views = []
    for i in range(len(image_names)):
        t_idx = i // num_views  # Time index (which frame in sequence)
        cam_idx = i % num_views  # Camera index (which view)
        views.append({
            "img": images[i].unsqueeze(0), # (1, C, H, W)
            "view_idxs": torch.tensor([[cam_idx, t_idx]], device=device, dtype=torch.long)
        })

    inference_start = time.time()
    
    with torch.no_grad():
        with torch.amp.autocast('cuda'):
            predictions = model.inference(views=views)
    
    inference_time = time.time() - inference_start
    print(f"✓ Inference completed in {inference_time:.2f}s ({inference_time/len(image_names):.2f}s per frame)")

    pts_list = [pm["pts3d"].detach().cpu().numpy() for pm in predictions["pointmaps"]]
    conf_list = [pm["conf"].detach().cpu().numpy() for pm in predictions["pointmaps"]]
    
    # Get pose encodings if available (VDPM extra)
    pose_enc = None
    if "pose_enc" in predictions:
        pose_enc = predictions["pose_enc"].detach().cpu().numpy()
    
    del predictions
    if device == "cuda":
        torch.cuda.empty_cache()

    world_points_raw = np.concatenate(pts_list, axis=0)  # (T, S, H, W, 3)
    world_points_conf_raw = np.concatenate(conf_list, axis=0)  # (T, S, H, W)
    
    T = world_points_raw.shape[0]
    S = world_points_raw.shape[1]
    num_timesteps = T
    
    if num_views > 1 and S == num_views * T:
        # Multi-view case
        print(f"DEBUG: Multi-view mode - extracting ALL {num_views} views")
        world_points_list = []
        world_points_conf_list = []
        for t in range(T):
            start_idx = t * num_views
            end_idx = start_idx + num_views
            world_points_list.append(world_points_raw[t, start_idx:end_idx])
            world_points_conf_list.append(world_points_conf_raw[t, start_idx:end_idx])
        
        world_points_mv = np.stack(world_points_list, axis=0)
        world_points_conf_mv = np.stack(world_points_conf_list, axis=0)
        
        world_points_full = world_points_mv
        world_points_conf_full = world_points_conf_mv
    else:
        # Single-view or fallback
        if world_points_raw.ndim == 5 and world_points_raw.shape[0] == 1:
            world_points = world_points_raw[0]
            world_points_conf = world_points_conf_raw[0]
        elif world_points_raw.ndim == 5:
            world_points_list = []
            world_points_conf_list = []
            for t in range(min(T, S)):
                world_points_list.append(world_points_raw[t, t])
                world_points_conf_list.append(world_points_conf_raw[t, t])
            world_points = np.stack(world_points_list, axis=0)
            world_points_conf = np.stack(world_points_conf_list, axis=0)
        else:
            world_points = world_points_raw
            world_points_conf = world_points_conf_raw
        
        world_points_full = world_points
        world_points_conf_full = world_points_conf

    # Save tracks
    tracks_path = os.path.join(target_dir, "tracks.npz")
    print(f"Saving tracks (clean) to {tracks_path}")
    np.savez_compressed(
        tracks_path,
        world_points=world_points_full,
        world_points_conf=world_points_conf_full,
        num_views=num_views,
        num_timesteps=num_timesteps
    )
    
    if pose_enc is not None:
        poses_path = os.path.join(target_dir, "poses.npz")
        print(f"Saving poses to {poses_path}")
        np.savez_compressed(poses_path, pose_enc=pose_enc)

    # ========================================================================
    # COMPUTE AND SAVE DEPTHS
    # ========================================================================
    depths = None
    if pose_enc is not None:
        print("Computing depth maps from world points and camera poses...")
        # Get image dimensions from world_points shape
        # world_points_full is (T, V, H, W, 3)
        if world_points_full.ndim == 5:
            _, _, H, W, _ = world_points_full.shape
        elif world_points_full.ndim == 4:
            # Single view: (T, H, W, 3)
            _, H, W, _ = world_points_full.shape
            world_points_full = world_points_full[:, np.newaxis, :, :, :]  # Add view dimension
        else:
            H, W = 518, 518  # Fallback
            print(f"Warning: Unexpected world_points shape {world_points_full.shape}")
        
        extrinsics, intrinsics = decode_poses(pose_enc, (H, W))
        depths = compute_depths(world_points_full, extrinsics, num_views)
        
        # Save depths as npz
        depths_path = os.path.join(target_dir, "depths.npz")
        print(f"Saving depths to {depths_path}")
        np.savez_compressed(
            depths_path,
            depths=depths,
            num_views=num_views,
            num_timesteps=num_timesteps
        )
        
        # Save individual depth images as PNGs
        depths_dir = os.path.join(target_dir, "depths")
        os.makedirs(depths_dir, exist_ok=True)
        print(f"Saving depth images to {depths_dir}/")
        
        T_depth = depths.shape[0]
        V_depth = depths.shape[1]
        for t in range(T_depth):
            for v in range(V_depth):
                depth_map = depths[t, v]
                png_path = os.path.join(depths_dir, f"depth_t{t:04d}_v{v:02d}.png")
                write_depth_to_png(png_path, depth_map)
        
        print(f"✓ Saved {T_depth * V_depth} depth images")
    else:
        print("⚠ No pose encodings available - skipping depth computation")

    # Save output_4d (for consistency with Gradio)
    output_path = os.path.join(target_dir, "output_4d.npz")
    save_dict = {
        "world_points": world_points_full,
        "world_points_conf": world_points_conf_full,
        "timestamps": np.arange(num_timesteps),
        "num_views": num_views,
        "num_timesteps": num_timesteps
    }
    if depths is not None:
        save_dict["depths"] = depths
    np.savez_compressed(output_path, **save_dict)
    
    return {
        "tracks_path": tracks_path,
        "output_path": output_path,
        "depths_path": os.path.join(target_dir, "depths.npz") if depths is not None else None
    }

def main():
    parser = argparse.ArgumentParser(description="Run VDPM Inference (CLI)")
    parser.add_argument("--input", required=True, help="Input video file or folder containing videos")
    parser.add_argument("--output", required=True, help="Output directory")
    parser.add_argument("--name", help="Optional name for the reconstruction folder")
    
    args = parser.parse_args()
    
    input_path = Path(args.input)
    output_root = Path(args.output)
    
    # 1. Gather Videos
    videos = []
    if input_path.is_file():
        videos = [str(input_path)]
    elif input_path.is_dir():
        # Find all videos in folder
        found_videos = set()
        for ext in ['*.mp4', '*.mov', '*.avi', '*.mkv']:
             # Case-insensitive recursive glob is hard, sticking to simple globs
             matches = glob.glob(str(input_path / ext)) + glob.glob(str(input_path / ext.upper()))
             for m in matches:
                 found_videos.add(os.path.abspath(m))
        
        # Sort to ensure order (cam1, cam2...) matches Gradio behavior
        videos = sorted(list(found_videos))
        
        if not videos:
            print(f"No videos found in {input_path}")
            return
            
        print(f"Found {len(videos)} videos in {input_path}")
    else:
        print(f"Input {input_path} not found")
        return

    # 2. Setup Output Directory
    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
    folder_name = args.name if args.name else f"reconstruction_{timestamp}"
    target_dir = output_root / folder_name
    target_dir_images = target_dir / "images"
    
    if target_dir.exists():
        print(f"Cleaning existing output dir: {target_dir}")
        shutil.rmtree(target_dir)
    target_dir_images.mkdir(parents=True, exist_ok=True)
    
    # 3. Extract Frames (Interleaved)
    process_videos_interleaved(videos, str(target_dir_images))
    
    # 4. Save Metadata
    num_views = len(videos)
    with open(target_dir / "meta.json", "w") as f:
        json.dump({"num_views": num_views}, f)
        
    print(f"Metadata saved: {num_views} view(s)")
    
    # 5. Load Model and Run
    print("Loading model...")
    cfg = load_cfg_from_cli()
    model = load_model(cfg)
    
    print("Running inference...")
    run_model(str(target_dir), model)
    
    print(f"\n{'='*60}")
    print(f"Success! Output saved to:\n{target_dir}")
    print(f"Next step: Train Gaussian Splats using:")
    print(f"python gs/train_vdpm.py --input {target_dir} --output output/splats")
    print(f"{'='*60}")

if __name__ == "__main__":
    main()