src/inference.py

import torch, os, sys, glob
import argparse
import math
import numpy as np
from PIL import Image

sys.path.append(os.getcwd())
sys.path.append("./DepthAnything3/src")

from DepthAnything3.src.depth_anything_3.api import DepthAnything3

from diffsynth.core import ModelConfig
from diffsynth import load_state_dict
from src.MetaView_pipeline import MetaViewPipeline
import torch.nn.functional as F

def compute_target_extrinsic(yaw_deg, pitch_deg, radius):
    """
    Compute the camera extrinsic matrix (World-to-Camera) for rotation
    around a sphere center in front of the camera.
    Supports simultaneous yaw (left-right) and pitch (up-down) angles.

    Args:
        yaw_deg (float): Yaw angle in degrees.
        pitch_deg (float): Pitch angle in degrees.
        radius (float): Distance from the rotation sphere center to the camera
                        (typically the depth of the target object).
    Returns:
        numpy.ndarray: 4x4 extrinsic matrix.
    """
    yaw = np.radians(yaw_deg)
    pitch = np.radians(pitch_deg)
    
    # Rotation matrix around Y axis (Yaw)
    R_y = np.array([
        [np.cos(yaw),  0, np.sin(yaw)],
        [0,            1, 0          ],
        [-np.sin(yaw), 0, np.cos(yaw)]
    ])
    
    # Rotation matrix around X axis (Pitch)
    R_x = np.array([
        [1, 0,             0            ],
        [0, np.cos(pitch), -np.sin(pitch)],
        [0, np.sin(pitch), np.cos(pitch) ]
    ])
    
    # Combined rotation (pitch first, then yaw)
    R = R_y @ R_x
    
    # Set sphere center coordinates C
    C = np.array([0.0, 0.0, radius])
    
    # Compute translation vector t = C - R * C
    t = C - R @ C
    
    # Construct 4x4 extrinsic matrix
    T = np.eye(4)
    T[:3, :3] = R
    T[:3, 3] = t
    
    return T

def main():
    parser = argparse.ArgumentParser(description="MetaView Interactive Inference CLI")
    
    # Core interactive parameters
    parser.add_argument("--image_path", type=str, required=True, help="Path to the input image")
    parser.add_argument("--output_path", type=str, default="./output_novel_view.png", help="Path to save the generated image")
    parser.add_argument("--yaw", type=float, default=0.0, help="Yaw angle in degrees (e.g., 60 for right, -60 for left)")
    parser.add_argument("--pitch", type=float, default=0.0, help="Pitch angle in degrees (e.g., 30 for top, -30 for bottom)")
    parser.add_argument("--radius", type=float, default=None, help="Rotation radius. If None, auto-calculated from center depth.")
    
    # Model path parameters
    parser.add_argument("--da3_giant_path", type=str, default="../../Depth-Anything-3/model/DA3-GIANT-1.1", help="Path to DA3 Giant")
    parser.add_argument("--da3_depth_path", type=str, default="../../Depth-Anything-3/model/DA3NESTED-GIANT-LARGE-1.1", help="Path to DA3 Depth model")
    parser.add_argument("--qwen_path", type=str, default=None, help="Base path to Qwen-Image-Edit models")
    parser.add_argument("--ckpt_path", type=str, required=True, help="Path to the trained MetaView checkpoint (.safetensors)")
    
    args = parser.parse_args()

    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    
    # Global parameter configuration
    export_3D_feat_layers = [19, 27, 33, 39]
    prope_dim_arrange = [64, 20, 20, 24]
    add_depth = (len(prope_dim_arrange) == 4)
    merge_3D = True
    prompt = ["镜头视角转到指定位置"]

    # 1. Load input image
    print(f"[*] Loading input image from {args.image_path}...")
    original_image = Image.open(args.image_path).convert("RGB")
    edit_image = original_image.resize((960, 528))
    
    # =========================================================================
    # 2. Depth and geometry prior extraction (Depth & Intrinsics)
    # =========================================================================
    print("[*] Loading DepthAnything3 Prior Models...")
    
    with torch.inference_mode():
        # Load feature extraction model (GIANT)
        model_3D = DepthAnything3.from_pretrained(args.da3_giant_path).to(device=device)
        
        print("    -> Extracting 3D Features and Intrinsics...")
        feat_3D_output = model_3D.inference([edit_image], export_feat_layers=export_3D_feat_layers, process_res=840)
        
        # Process intrinsics Ks
        intri = feat_3D_output.intrinsics[0]
        width = intri[0, 2] * 2
        height = intri[1, 2] * 2
        Ks_matrix = [
            [intri[0, 0] / width, 0.0,                  0.0],
            [0.0,                 intri[1, 1] / height, 0.0],
            [0.0,                 0.0,                  1.0],
        ]
        Ks = torch.Tensor(Ks_matrix)
        Ks = torch.stack([Ks, Ks], dim=0).unsqueeze(0) # Shape: (1, 2, 3, 3)
        
        # Process features feat_3D
        feats = [torch.from_numpy(feat_3D_output.aux[f"feat_layer_{layer}"]) for layer in export_3D_feat_layers]
        feat_3D = torch.cat(feats, dim=-1).to(dtype=torch.bfloat16, device=device)
        
        # Release feature extraction model
        model_3D.to("cpu")
        del model_3D
        torch.cuda.empty_cache()

        # Load depth extraction model (NESTED)
        print("    -> Extracting Depth Map...")
        model_depth = DepthAnything3.from_pretrained(args.da3_depth_path).to(device=device)
        prediction = model_depth.inference([edit_image], process_res=840)
        
        depth_edit = torch.Tensor(prediction.depth).unsqueeze(0)
        depth_edit = F.interpolate(depth_edit, size=(528, 960), mode='bilinear', align_corners=False)[0]
        depth_latent = torch.zeros_like(depth_edit)
        depth = torch.cat([depth_latent, depth_edit], dim=0).unsqueeze(0) # Shape: (1, 2, H, W)
        
        # Release depth model
        model_depth.to("cpu")
        del model_depth
        torch.cuda.empty_cache()

    # =========================================================================
    # 3. Target pose calculation
    # =========================================================================
    # Auto-derive radius: if user did not specify radius, use center depth from the depth map
    if args.radius is None:
        depth_squeeze = depth[0, 1] # Get real depth channel
        z_c = depth_squeeze[depth_squeeze.shape[0]//2, depth_squeeze.shape[1]//2].item()
        args.radius = z_c
        print(f"[*] Auto-calculated rotation radius from center depth: {args.radius:.4f}")
        
    print(f"[*] Calculating Target Pose -> Yaw: {args.yaw}°, Pitch: {args.pitch}°, Radius: {args.radius}")
    extrinsic_target = compute_target_extrinsic(args.yaw, args.pitch, args.radius)
    extrinsic_source = np.eye(4)
    
    # Construct viewmats tensor: Shape (1, 2, 4, 4) -> [Target, Source]
    viewmats = torch.Tensor(np.stack((extrinsic_target, extrinsic_source), axis=0)).unsqueeze(0)

    # =========================================================================
    # 4. Generation model loading and inference (DiT Pipeline)
    # =========================================================================
    print("[*] Loading Qwen-Image-Edit Pipeline...")

    if args.qwen_path:
        print(f"[*] Loading Qwen-Image-Edit from {args.qwen_path}")
        pipe = MetaViewPipeline.from_pretrained(
            torch_dtype=torch.bfloat16,
            device="cuda",
            model_configs=[
                ModelConfig(path=glob.glob(f"{args.qwen_path}/Qwen-Image-Edit/transformer/diffusion_pytorch_model*.safetensors")),
                ModelConfig(path=glob.glob(f"{args.qwen_path}/Qwen-Image-Edit/text_encoder/model*.safetensors")),
                ModelConfig(path=glob.glob(f"{args.qwen_path}/Qwen-Image-Edit/vae/diffusion_pytorch_model.safetensors")),
            ],
            tokenizer_config=None,
            processor_config=ModelConfig(path=f"{args.qwen_path}/Qwen-Image-Edit/processor/"),
        )
    else:   # Auto download
        pipe = MetaViewPipeline.from_pretrained(
        torch_dtype=torch.bfloat16,
        device="cuda",
        model_configs=[
            ModelConfig(model_id="Qwen/Qwen-Image-Edit", origin_file_pattern="transformer/diffusion_pytorch_model*.safetensors"),
            ModelConfig(model_id="Qwen/Qwen-Image", origin_file_pattern="text_encoder/model*.safetensors"),
            ModelConfig(model_id="Qwen/Qwen-Image", origin_file_pattern="vae/diffusion_pytorch_model.safetensors"),
        ],
        tokenizer_config=None,
        processor_config=ModelConfig(model_id="Qwen/Qwen-Image-Edit", origin_file_pattern="processor/"),
    )

    print(f"[*] Loading MetaView Weights from {args.ckpt_path}...")
    state_dict = load_state_dict(args.ckpt_path)
    pipe.dit.load_state_dict(state_dict, strict=False)

    print("[*] Starting Generation (40 Steps)...")
    with torch.inference_mode():
        generated_image = pipe(
            prompt, edit_image=edit_image, edit_image_auto_resize=False,
            seed=0,
            viewmats=viewmats.to(device=device, dtype=torch.bfloat16),
            Ks=Ks.to(device=device, dtype=torch.bfloat16),
            prope_dim_arrange=prope_dim_arrange,
            add_attn=True,
            add_3D=True,
            feat_3D=feat_3D,
            depth=depth.to(device=device, dtype=torch.bfloat16) if add_depth else None,
            merge_3D=merge_3D,
            val=True,
            num_inference_steps=40,
            height=528, width=960,
        )

    # =========================================================================
    # 5. Save result (stitch source and generated images for comparison)
    # =========================================================================
    stitched_image = Image.new('RGB', (960 * 2, 528), (255, 255, 255))
    stitched_image.paste(edit_image, (0, 0))
    stitched_image.paste(generated_image, (960, 0))
    
    os.makedirs(os.path.dirname(os.path.abspath(args.output_path)), exist_ok=True)
    stitched_image.save(args.output_path)
    print(f"Success! Result saved to {args.output_path}")

if __name__ == '__main__':
    main()