generate.py

import torch
from PIL import Image
from shap_e.diffusion.sample import sample_latents
from shap_e.models.download import load_model # Used for loading 'transmitter', 'text300M', 'image300M'
from shap_e.util.notebooks import decode_latent_mesh, create_pan_cameras, render_views
# You might also need this if you are handling raw diffusion setup
# from shap_e.diffusion.gaussian_diffusion import diffusion_from_config
# from shap_e.models.configs import model_from_config # for more explicit config loading

# Determine device (CPU for Spaces without GPU, or CUDA if available)
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

# Load models
# The 'transmitter' model includes the image encoder (CLIP) and is used for image input.
# The 'image300M' model is the diffusion model for image-conditioned generation.
# The 'text300M' model is the diffusion model for text-conditioned generation.
# Load these once globally to avoid repeated loading.
print(f"Loading models on {device}...")
xm = load_model('transmitter', device=device) # Contains the image encoder (e.g., CLIP)
model_text = load_model('text300M', device=device)
model_image = load_model('image300M', device=device) # This is the diffusion model for image input

# Diffusion configuration is often loaded automatically or integrated into the pipeline
# diffusion = diffusion_from_config(load_config('diffusion')) # If you need explicit diffusion object

print("Models loaded successfully.")


def generate_model_from_text(prompt: str, filename="model.glb"):
    print(f"Generating 3D model from text: '{prompt}'")
    # `sample_latents` takes the model directly for text-to-3D
    latents = sample_latents(
        batch_size=1, # Generate one model
        model=model_text,
        diffusion=None, # Diffusion is often integrated or defaulted in sample_latents when using models from load_model
        guidance_scale=15.0,
        model_kwargs=dict(texts=[prompt]),
        progress=True,
        clip_denoised=True,
        use_fp16=True if device.type == 'cuda' else False, # Use FP16 only if GPU
        use_karras=True,
        karras_steps=64,
        sigma_min=1e-3,
        sigma_max=160,
        s_churn=0,
    )
    print("Latents sampled.")

    mesh = decode_latent_mesh(xm, latents[0]).tri_mesh() # Use xm (transmitter) to decode
    mesh.export(filename)
    print(f"Model saved to {filename}")
    return filename

def generate_model_from_image(image: Image.Image, filename="model.glb"):
    print("Generating 3D model from image...")

    # For image input, you need to prepare the image.
    # The 'transmitter' (xm) implicitly handles the CLIP embedding when used in sample_latents
    # with `is_image=True` and `model_kwargs=dict(images=[image_tensor])`.
    # First, resize the image to the expected input size (e.g., 256x256) and convert to tensor.
    # Note: shap-e expects a specific image format (likely 3x256x256 float32 tensor in [0,1])
    # You might need torchvision.transforms or similar for robust image preprocessing.
    from torchvision import transforms
    preprocess = transforms.Compose([
        transforms.Resize(256),
        transforms.CenterCrop(256),
        transforms.ToTensor(), # Converts PIL Image to torch.Tensor (C, H, W) in [0, 1]
    ])
    image_tensor = preprocess(image).unsqueeze(0).to(device) # Add batch dimension (1, C, H, W)

    latents = sample_latents(
        batch_size=1,
        model=model_image, # Use the image-conditioned model
        diffusion=None, # Same as above, often integrated
        guidance_scale=3.0, # Guidance scale often different for image-to-3D
        model_kwargs=dict(images=[image_tensor]), # Pass the tensor
        progress=True,
        clip_denoised=True,
        use_fp16=True if device.type == 'cuda' else False,
        use_karras=True,
        karras_steps=64,
        sigma_min=1e-3,
        sigma_max=160,
        s_churn=0,
    )
    print("Latents sampled.")

    mesh = decode_latent_mesh(xm, latents[0]).tri_mesh() # Use xm (transmitter) to decode
    mesh.export(filename)
    print(f"Model saved to {filename}")
    return filename