infer_custom.py

#!/usr/bin/env python3
"""
Batch RGB → TIR inference for TherA.

Recommended (no LLaVA weights needed): use a reference .pt cache
  python infer_custom.py --rgb-dir ./examples/rgb --output-dir ./preds \\
      --reference-cache weights/reference_caches/SUNNY.pt

On-the-fly LLaVA (optional): requires separate LLaVA checkpoints
  python infer_custom.py --rgb-dir ./examples/rgb --output-dir ./preds \\
      --llava-base-path weights/llava/llava-1.5-7b-hf \\
      --llava-lora-path weights/llava/llava-miragehd-lora

Single-image example-guided mode: see infer_example_guided.py
"""

from __future__ import annotations

import argparse
from pathlib import Path
from typing import List

import torch
from PIL import Image
from tqdm import tqdm
from contextlib import nullcontext

from thera_paths import (
    DEFAULT_CHECKPOINT,
    DEFAULT_MERGED_MODEL,
    DEFAULT_PRETRAINED_SD,
    setup_project_path,
)
from thera_llava import create_frozen_llava_extractor

setup_project_path()

from diffusers import AutoencoderKL, DDIMScheduler, UNet2DConditionModel
from torchvision import transforms
from torchvision.utils import save_image

from lavi_ip2p.llava_adapter import create_llava_adapter
from lavi_ip2p.unet_8ch import convert_unet_to_8ch

IMAGE_EXTENSIONS = {".jpg", ".jpeg", ".png", ".bmp", ".tif", ".tiff"}


def parse_args():
    parser = argparse.ArgumentParser(description="Generate TIR predictions from custom RGB images")
    
    # Model paths (defaults under weights/ — see README.md)
    parser.add_argument(
        "--checkpoint",
        type=str,
        default=str(DEFAULT_CHECKPOINT),
        help="Directory containing model.pt (TherA UNet + adapter weights)",
    )
    parser.add_argument(
        "--merged-model-path",
        type=str,
        default=str(DEFAULT_MERGED_MODEL),
        help="Directory with unet/ and adapter/ architecture configs",
    )
    parser.add_argument(
        "--pretrained-sd",
        type=str,
        default=str(DEFAULT_PRETRAINED_SD),
        help="Stable Diffusion folder with vae/ and scheduler/ subfolders",
    )
    
    # LLaVA paths (only for on-the-fly feature extraction)
    parser.add_argument(
        "--llava-base-path",
        type=str,
        default=None,
        help="Base LLaVA model. Required unless --reference-cache or --cache-dir is set.",
    )
    parser.add_argument(
        "--llava-lora-path",
        type=str,
        default=None,
        help="Optional LLaVA LoRA weights (on-the-fly mode only)",
    )
    parser.add_argument("--llava-prompt", type=str, 
                       default="How would this RGB scene appear in long-wave thermal infrared spectrum",
                       help="Prompt for LLaVA feature extraction")
    
    # Reference cache mode (alternative to on-the-fly extraction)
    parser.add_argument("--reference-cache", type=str, default=None,
                       help="Path to a reference .pt cache file. If provided, uses this fixed feature for all images instead of extracting per-image.")
    parser.add_argument("--cache-dir", type=str, default=None,
                       help="Directory with per-image .pt cache files (matched by filename stem). Alternative to --reference-cache.")
    
    # Data
    parser.add_argument("--rgb-dir", type=str, required=True,
                       help="Directory containing RGB images")
    parser.add_argument("--output-dir", type=str, default="custom_predictions",
                       help="Output directory for predictions")
    parser.add_argument("--recursive", action="store_true",
                       help="Search for images recursively in subdirectories")
    
    # Sampling
    parser.add_argument("--num-steps", type=int, default=100, help="DDIM sampling steps")
    parser.add_argument("--cfg-text", type=float, default=3.5, help="CFG scale for text")
    parser.add_argument("--cfg-image", type=float, default=1.5, help="CFG scale for image")
    parser.add_argument("--seed", type=int, default=42, help="Random seed")
    
    # Processing
    parser.add_argument("--target-size", type=int, default=None,
                       help="Resize images to this size (default: keep original, round to 32)")
    parser.add_argument("--batch-size", type=int, default=1,
                       help="Batch size (>1 for faster processing if you have enough VRAM)")
    
    # Device
    parser.add_argument("--device", type=str, default="cuda", help="Device (cuda/cpu)")
    
    return parser.parse_args()


def load_models(args):
    """Load VAE, UNet, Adapter, Scheduler, and optionally LLaVA Extractor"""
    print("\n" + "="*80)
    print(f"Loading models on device {args.device}...")
    print("="*80)
    
    device = torch.device(args.device)
    
    # Determine if we need LLaVA extractor
    use_cache = args.reference_cache is not None or args.cache_dir is not None
    
    # 1. VAE
    print("\n[1/5] Loading VAE...")
    vae = AutoencoderKL.from_pretrained(args.pretrained_sd, subfolder="vae")
    vae.requires_grad_(False)
    vae.eval()
    vae.to(device)
    print("✓ VAE loaded")
    
    # 2. Scheduler
    print("\n[2/5] Loading scheduler...")
    scheduler = DDIMScheduler.from_pretrained(args.pretrained_sd, subfolder="scheduler")
    print("✓ Scheduler loaded")
    
    # 3. UNet (load from checkpoint)
    checkpoint_dir = Path(args.checkpoint)
    print(f"\n[3/5] Loading UNet from checkpoint: {checkpoint_dir}")
    checkpoint_path = checkpoint_dir / "model.pt"
    if not checkpoint_path.exists():
        raise FileNotFoundError(f"Checkpoint not found: {checkpoint_path}")
    
    checkpoint = torch.load(checkpoint_path, map_location="cpu")
    
    # Load base UNet architecture from merged model
    merged_unet_path = Path(args.merged_model_path) / "unet"
    unet = UNet2DConditionModel.from_pretrained(str(merged_unet_path))
    
    # Convert to 8-channel for image-to-image task
    print("  Converting UNet to 8-channel input...")
    unet = convert_unet_to_8ch(unet)
    
    # Load trained weights
    unet.load_state_dict(checkpoint['unet'])
    unet.requires_grad_(False)
    unet.eval()
    unet.to(device)
    
    global_step = checkpoint.get('global_step', 'unknown')
    epoch = checkpoint.get('epoch', 'unknown')
    print(f"✓ UNet loaded (step={global_step}, epoch={epoch})")
    
    # 4. Adapter
    print("\n[4/5] Loading LLaVA Adapter...")
    merged_adapter_path = Path(args.merged_model_path) / "adapter"
    adapter = create_llava_adapter(
        adapter_path=str(merged_adapter_path),
        use_rms_norm=False,
        learnable_scale=False,
        init_scale=1.0,
        freeze_adapter=True,
    )
    
    # Load trained adapter weights if available in checkpoint
    if 'llava_adapter' in checkpoint:
        adapter.load_state_dict(checkpoint['llava_adapter'])
        print("✓ Adapter loaded from checkpoint")
    else:
        print("✓ Adapter loaded (using base weights)")
    
    adapter.requires_grad_(False)
    adapter.eval()
    adapter.to(device)
    
    # 5. LLaVA Extractor (optional - only if not using cache)
    llava_extractor = None
    if use_cache:
        print("\n[5/5] Skipping LLaVA extractor (using cached features)")
    else:
        print("\n[5/5] Loading LLaVA Feature Extractor...")
        if not args.llava_base_path:
            raise ValueError("--llava-base-path is required when not using --reference-cache or --cache-dir")
        llava_extractor = create_frozen_llava_extractor(
            llava_base_path=args.llava_base_path,
            llava_lora_path=args.llava_lora_path,
            device=args.device,
            load_8bit=False,
            load_4bit=False,
            merge_lora=True,
        )
        print("✓ LLaVA extractor loaded")
    
    return vae, unet, adapter, scheduler, llava_extractor, device


def find_images(rgb_dir: Path, recursive: bool = False) -> List[Path]:
    """Find all image files in directory"""
    if recursive:
        images = []
        for ext in IMAGE_EXTENSIONS:
            images.extend(rgb_dir.rglob(f"*{ext}"))
            images.extend(rgb_dir.rglob(f"*{ext.upper()}"))
        return sorted(set(images))
    else:
        images = []
        for ext in IMAGE_EXTENSIONS:
            images.extend(rgb_dir.glob(f"*{ext}"))
            images.extend(rgb_dir.glob(f"*{ext.upper()}"))
        return sorted(set(images))


def load_and_prepare_image(image_path: Path, target_size: int = None, device: torch.device = None):
    """Load RGB image and prepare for processing"""
    rgb_img = Image.open(image_path).convert('RGB')
    rgb_w, rgb_h = rgb_img.size
    
    # Determine target dimensions
    if target_size is not None:
        target_w = target_h = target_size
    else:
        # Ensure dimensions divisible by 32 (required for VAE)
        target_w = ((rgb_w + 31) // 32) * 32
        target_h = ((rgb_h + 31) // 32) * 32
    
    # Resize if needed
    if target_w != rgb_w or target_h != rgb_h:
        rgb_img = rgb_img.resize((target_w, target_h), Image.Resampling.LANCZOS)
    
    # Convert to tensor for diffusion model
    transform = transforms.Compose([
        transforms.ToTensor(),
        transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5]),  # [-1, 1]
    ])
    
    rgb_tensor = transform(rgb_img).unsqueeze(0)
    if device is not None:
        rgb_tensor = rgb_tensor.to(device)
    
    return rgb_tensor, rgb_img  # Return both tensor and PIL image


def load_llava_cache(cache_path: Path, device: torch.device):
    """Load a cached LLaVA hidden state from .pt file"""
    try:
        llava_hidden = torch.load(cache_path, map_location="cpu", weights_only=True)
    except TypeError:
        # For older torch versions without weights_only
        llava_hidden = torch.load(cache_path, map_location="cpu")
    
    if not isinstance(llava_hidden, torch.Tensor):
        llava_hidden = torch.tensor(llava_hidden)
    
    # Add batch dimension if needed
    if llava_hidden.ndim == 2:
        llava_hidden = llava_hidden.unsqueeze(0)
    
    return llava_hidden.to(device)


def find_cache_for_image(image_path: Path, cache_dir: Path) -> Path:
    """Find matching cache file for an image based on filename stem"""
    stem = image_path.stem
    for ext in ['.pt', '.pth']:
        cache_path = cache_dir / f"{stem}{ext}"
        if cache_path.exists():
            return cache_path
    raise FileNotFoundError(f"No cache file found for {image_path.name} in {cache_dir}")


def create_null_text(llava_tokens):
    """Create null tokens for unconditional generation"""
    return torch.zeros_like(llava_tokens)


@torch.no_grad()
def translate_image(
    vae,
    unet,
    adapter,
    scheduler,
    rgb_tensor,
    llava_hidden,
    num_steps=100,
    cfg_text=3.5,
    cfg_image=1.5,
    device=torch.device("cuda"),
):
    """Translate RGB to TIR using LLaVA features + conditional generation with CFG"""
    
    # Encode RGB
    rgb_latents = vae.encode(rgb_tensor).latent_dist.mode() * vae.config.scaling_factor
    
    # Process LLaVA features through adapter
    use_autocast = device.type == "cuda"
    llava_input = llava_hidden.to(device)
    if use_autocast:
        llava_input = llava_input.to(dtype=torch.bfloat16)
    
    autocast_ctx = (
        torch.autocast(device_type=device.type, dtype=torch.bfloat16) 
        if use_autocast else nullcontext()
    )
    with autocast_ctx:
        llava_tokens = adapter(llava_input)
    llava_tokens = llava_tokens.to(device=device, dtype=next(unet.parameters()).dtype)
    
    # Create null tokens for CFG
    null_tokens = create_null_text(llava_tokens)
    
    # Initialize random noise
    latent_shape = (1, 4, rgb_latents.shape[2], rgb_latents.shape[3])
    latents = torch.randn(latent_shape, device=device, dtype=rgb_latents.dtype)
    
    # Setup scheduler
    scheduler.set_timesteps(num_steps, device=device)
    
    zeros_rgb = torch.zeros_like(rgb_latents)
    
    for t in scheduler.timesteps:
        t_batch = torch.full((1,), int(t), device=device, dtype=torch.long)
        
        # 3 forward passes for dual-CFG
        x8_full = torch.cat([latents, rgb_latents], dim=1)
        x8_no_img = torch.cat([latents, zeros_rgb], dim=1)
        
        # Full conditioning
        eps_full = unet(x8_full, t_batch, encoder_hidden_states=llava_tokens).sample
        
        # No text (image only)
        eps_no_text = unet(x8_full, t_batch, encoder_hidden_states=null_tokens).sample
        
        # No image (text only)
        eps_no_img = unet(x8_no_img, t_batch, encoder_hidden_states=llava_tokens).sample
        
        # Combine with dual-CFG
        eps = eps_full + cfg_text * (eps_full - eps_no_text) + cfg_image * (eps_full - eps_no_img)
        eps = torch.clamp(eps, -5.0, 5.0)  # Safety guard
        
        latents = scheduler.step(eps, t, latents, eta=0.0).prev_sample
    
    # Decode
    pred_tir = vae.decode(latents / vae.config.scaling_factor).sample
    pred_tir = (pred_tir / 2 + 0.5).clamp(0, 1)
    
    return pred_tir


def main():
    args = parse_args()
    
    # Set seed
    torch.manual_seed(args.seed)
    if torch.cuda.is_available():
        torch.cuda.manual_seed_all(args.seed)
    
    # Determine processing mode
    use_reference = args.reference_cache is not None
    use_cache_dir = args.cache_dir is not None
    
    if use_reference and use_cache_dir:
        raise ValueError("Cannot use both --reference-cache and --cache-dir. Choose one.")
    
    print("\n" + "="*80)
    if use_reference:
        print("Custom Dataset Inference (Reference Cache Mode)")
    elif use_cache_dir:
        print("Custom Dataset Inference (Per-Image Cache Mode)")
    else:
        print("Custom Dataset Inference (On-the-fly LLaVA Features)")
    print("="*80)
    print(f"RGB directory: {args.rgb_dir}")
    print(f"Checkpoint: {args.checkpoint}")
    
    if use_reference:
        print(f"Reference cache: {args.reference_cache}")
    elif use_cache_dir:
        print(f"Cache directory: {args.cache_dir}")
    else:
        print(f"LLaVA base: {args.llava_base_path}")
        if args.llava_lora_path:
            print(f"LLaVA LoRA: {args.llava_lora_path}")
        print(f"Prompt: {args.llava_prompt}")
    
    print(f"Output directory: {args.output_dir}")
    print(f"DDIM steps: {args.num_steps}")
    print(f"CFG scales: text={args.cfg_text}, image={args.cfg_image}")
    print("="*80)
    
    # Create output directory
    output_dir = Path(args.output_dir)
    output_dir.mkdir(parents=True, exist_ok=True)
    
    # Load models
    vae, unet, adapter, scheduler, llava_extractor, device = load_models(args)
    
    # Load reference cache if provided
    reference_hidden = None
    if use_reference:
        print(f"\nLoading reference cache from: {args.reference_cache}")
        reference_hidden = load_llava_cache(Path(args.reference_cache), device)
        print(f"✓ Reference cache loaded: {reference_hidden.shape}")
    
    # Validate cache directory if provided
    cache_dir = None
    if use_cache_dir:
        cache_dir = Path(args.cache_dir)
        if not cache_dir.exists():
            raise FileNotFoundError(f"Cache directory not found: {cache_dir}")
        print(f"\nUsing cache directory: {cache_dir}")
    
    # Find all RGB images
    rgb_dir = Path(args.rgb_dir)
    if not rgb_dir.exists():
        raise FileNotFoundError(f"RGB directory not found: {rgb_dir}")
    
    image_files = find_images(rgb_dir, recursive=args.recursive)
    print(f"\nFound {len(image_files)} images")
    
    if len(image_files) == 0:
        print("No images found. Exiting.")
        return
    
    # Process each image
    print("\n" + "="*80)
    print("Generating predictions...")
    print("="*80)
    
    successful = 0
    failed = 0
    skipped = 0
    
    for img_path in tqdm(image_files, desc="Processing images"):
        try:
            # Load and prepare RGB image
            rgb_tensor, rgb_pil = load_and_prepare_image(
                img_path, 
                target_size=args.target_size,
                device=device
            )
            
            # Get LLaVA hidden states based on mode
            if use_reference:
                # Use the same reference cache for all images
                llava_hidden = reference_hidden
            elif use_cache_dir:
                # Find matching cache file for this image
                try:
                    cache_path = find_cache_for_image(img_path, cache_dir)
                    llava_hidden = load_llava_cache(cache_path, device)
                except FileNotFoundError as e:
                    print(f"\nWarning: {e}")
                    skipped += 1
                    continue
            else:
                # Extract features on-the-fly using LLaVA
                llava_hidden = llava_extractor.extract_hidden_states([rgb_pil], [args.llava_prompt])
            
            # Translate RGB to TIR
            pred_tir = translate_image(
                vae,
                unet,
                adapter,
                scheduler,
                rgb_tensor,
                llava_hidden,
                num_steps=args.num_steps,
                cfg_text=args.cfg_text,
                cfg_image=args.cfg_image,
                device=device,
            )
            
            # Save prediction
            # Preserve relative path structure if recursive
            if args.recursive:
                rel_path = img_path.relative_to(rgb_dir)
                save_path = output_dir / rel_path
            else:
                save_path = output_dir / img_path.name
            
            save_path.parent.mkdir(parents=True, exist_ok=True)
            save_image(pred_tir.cpu(), str(save_path))
            
            successful += 1
            
        except Exception as e:
            print(f"\nError processing {img_path}: {e}")
            import traceback
            traceback.print_exc()
            failed += 1
            continue
    
    # Print final results
    print("\n" + "="*80)
    print("INFERENCE COMPLETE")
    print("="*80)
    print(f"Total images: {len(image_files)}")
    print(f"Successful: {successful}")
    if skipped > 0:
        print(f"Skipped (no cache): {skipped}")
    print(f"Failed: {failed}")
    if len(image_files) > 0:
        print(f"Success rate: {successful/len(image_files)*100:.1f}%")
    print(f"Predictions saved to: {args.output_dir}")
    print("="*80)


if __name__ == "__main__":
    main()