import os os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True") import spaces # MUST come before torch / any CUDA-touching import import torch import torch.nn.functional as F import gradio as gr import numpy as np import cv2 import yaml import sys import tempfile from pathlib import Path from huggingface_hub import hf_hub_download from omegaconf import OmegaConf # Add the app root to sys.path so core.* and Utils are importable CODE_DIR = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, CODE_DIR) from core.utils.utils import InputPadder from Utils import vis_disparity, AMP_DTYPE MODEL_REPO = "nvidia/c-fast-foundationstereo" WEIGHTS_FILENAME = "model_best_bp2_serialize.pth" CFG_FILENAME = "cfg.yaml" def _load_model(): """Download the checkpoint and cfg from the HF Hub, then torch.load the pickled model.""" weights_path = hf_hub_download(MODEL_REPO, WEIGHTS_FILENAME, repo_type="model") cfg_path = hf_hub_download(MODEL_REPO, CFG_FILENAME, repo_type="model") with open(cfg_path, "r") as f: cfg: dict = yaml.safe_load(f) cfg.setdefault("valid_iters", 8) cfg.setdefault("max_disp", 192) cfg.setdefault("normalize", True) cfg.setdefault("cv_group", 8) cfg.setdefault("volume_dim", 28) args = OmegaConf.create(cfg) # Monkey-patch torch.load to allow weights_only=False for this pickled model _orig_load = torch.load torch.load = lambda *a, **k: _orig_load(*a, **{**k, "weights_only": k.get("weights_only", False)}) model = _orig_load(weights_path, map_location="cpu", weights_only=False) torch.load = _orig_load model.args = args model.cuda().eval() return model, args model, model_args = _load_model() model = model.to("cuda") @spaces.GPU(duration=60) def predict_disparity( left_image: np.ndarray, right_image: np.ndarray, valid_iters: int = 8, max_disp: int = 192, ) -> tuple: """Estimate disparity from a rectified binocular stereo image pair. Given a rectified left and right stereo image (RGB), produces a color-mapped disparity visualization alongside the raw 16-bit disparity map. Args: left_image: Rectified left stereo image (RGB), as a numpy array (H, W, 3). right_image: Rectified right stereo image (RGB), as a numpy array (H, W, 3). valid_iters: Number of GRU refinement iterations (more = better quality, less = faster). max_disp: Maximum disparity for volume encoding. 192 is sufficient for most scenes. Returns: A tuple of (color-mapped disparity visualization, 16-bit disparity map). """ if left_image is None or right_image is None: raise ValueError("Both left and right images are required.") # Ensure RGB img0 = left_image[..., :3] img1 = right_image[..., :3] if img0.dtype != np.uint8: img0 = (img0 * 255).astype(np.uint8) if img0.max() <= 1.0 else img0.astype(np.uint8) img1 = (img1 * 255).astype(np.uint8) if img1.max() <= 1.0 else img1.astype(np.uint8) H, W = img0.shape[:2] img0_ori = img0.copy() img1_ori = img1.copy() # Resize right to match left if needed if img1.shape[:2] != (H, W): img1 = cv2.resize(img1, (W, H)) img0_t = torch.as_tensor(img0).cuda().float()[None].permute(0, 3, 1, 2) img1_t = torch.as_tensor(img1).cuda().float()[None].permute(0, 3, 1, 2) padder = InputPadder(img0_t.shape, divis_by=32, force_square=False) img0_t, img1_t = padder.pad(img0_t, img1_t) model.args.valid_iters = valid_iters model.args.max_disp = max_disp with torch.amp.autocast("cuda", enabled=True, dtype=AMP_DTYPE): disp = model.forward(img0_t, img1_t, iters=valid_iters, test_mode=True, optimize_build_volume="pytorch1") disp = padder.unpad(disp.float()) disp_np = disp.data.cpu().numpy().reshape(H, W).clip(0, None) # Color-mapped visualization vis = vis_disparity(disp_np, color_map=cv2.COLORMAP_TURBO) # Side-by-side: left | right | disparity combined = np.concatenate([img0_ori, img1_ori, vis], axis=1) # Save 16-bit disparity to a temp file disp_16 = (disp_np * 256).clip(0, 65535).astype(np.uint16) tmp = tempfile.NamedTemporaryFile(suffix=".png", delete=False) cv2.imwrite(tmp.name, disp_16) return combined, tmp.name CSS = """ #col-container { max-width: 1100px; margin: 0 auto; } .dark .gradio-container { color: var(--body-text-color); } """ with gr.Blocks(theme=gr.themes.Citrus(), css=CSS) as demo: with gr.Column(elem_id="col-container"): gr.Markdown("# Fast-FoundationStereo: Real-Time Zero-Shot Stereo Disparity") gr.Markdown( "Upload a **rectified** binocular stereo image pair to estimate a disparity map. " "This is [nvidia/c-fast-foundationstereo](https://huggingface.co/nvidia/c-fast-foundationstereo), " "a 14.6M-parameter real-time stereo foundation model from NVIDIA (CVPR 2026)." ) with gr.Row(): with gr.Column(): left_input = gr.Image(label="Left Image (rectified RGB)", type="numpy") right_input = gr.Image(label="Right Image (rectified RGB)", type="numpy") with gr.Column(): disparity_vis = gr.Image(label="Disparity Visualization (left | right | color-mapped disparity)") disparity_raw = gr.Image(label="16-bit Disparity Map", type="filepath") with gr.Accordion("Advanced Settings", open=False): valid_iters_slider = gr.Slider( minimum=2, maximum=16, value=8, step=1, label="Valid Iterations (GRU refinement steps)", info="More iterations = better quality but slower. 8 is default, 4 is faster." ) max_disp_slider = gr.Slider( minimum=64, maximum=416, value=192, step=32, label="Max Disparity", info="Maximum disparity for volume encoding. 192 is enough for most scenes." ) run_btn = gr.Button("Estimate Disparity", variant="primary") gr.Examples( examples=[ ["examples/left.png", "examples/right.png", 8, 192], ["examples/left.png", "examples/right.png", 4, 192], ], inputs=[left_input, right_input, valid_iters_slider, max_disp_slider], outputs=[disparity_vis, disparity_raw], fn=predict_disparity, cache_examples=True, cache_mode="lazy", ) run_btn.click( fn=predict_disparity, inputs=[left_input, right_input, valid_iters_slider, max_disp_slider], outputs=[disparity_vis, disparity_raw], ) if __name__ == "__main__": demo.launch(mcp_server=True)