app.py

!pip install "huggingface_hub[hf_transfer]"
!pip install -U "huggingface_hub[cli]"
!pip install gradio trimesh scipy
!HF_HUB_ENABLE_HF_TRANSFER=1
!git clone https://github.com/PaulBorneP/MESA.git
!cd MESA
!mkdir weights
!huggingface-cli download NewtNewt/MESA --local-dir weights

import torch
from MESA.pipeline_terrain import TerrainDiffusionPipeline
import sys
import gradio as gr
import numpy as np
import trimesh
import tempfile
import torch
from scipy.spatial import Delaunay

sys.path.append('MESA/')

pipe = TerrainDiffusionPipeline.from_pretrained("./weights", torch_dtype=torch.float16)
pipe.to("cuda")

def generate_terrain(prompt, num_inference_steps, guidance_scale, seed, crop_size, prefix):
    """Generates terrain data (RGB and elevation) from a text prompt."""
    if prefix and not prefix.endswith(' '):
        prefix += ' '  # Ensure prefix ends with a space

    full_prompt = prefix + prompt
    generator = torch.Generator("cuda").manual_seed(seed)
    image, dem = pipe(full_prompt, num_inference_steps=num_inference_steps, guidance_scale=guidance_scale, generator=generator)

    # Center crop the image and dem
    h, w, c = image[0].shape
    start_h = (h - crop_size) // 2
    start_w = (w - crop_size) // 2
    end_h = start_h + crop_size
    end_w = start_w + crop_size

    cropped_image = image[0][start_h:end_h, start_w:end_w, :]
    cropped_dem = dem[0][start_h:end_h, start_w:end_w, :]

    return (255 * cropped_image).astype(np.uint8), 500*cropped_dem.mean(-1)

def create_3d_mesh(rgb, elevation):
    """Creates a 3D mesh from RGB and elevation data."""
    x, y = np.meshgrid(np.arange(elevation.shape[1]), np.arange(elevation.shape[0]))
    points = np.stack([x.flatten(), y.flatten()], axis=-1)
    tri = Delaunay(points)

    vertices = np.stack([x.flatten(), y.flatten(), elevation.flatten()], axis=-1)
    faces = tri.simplices

    mesh = trimesh.Trimesh(vertices=vertices, faces=faces, vertex_colors=rgb.reshape(-1, 3))

    return mesh

def generate_and_display(prompt, num_inference_steps, guidance_scale, seed, crop_size, prefix):
    """Generates terrain and displays it as a 3D model."""
    rgb, elevation = generate_terrain(prompt, num_inference_steps, guidance_scale, seed, crop_size, prefix)
    mesh = create_3d_mesh(rgb, elevation)

    with tempfile.NamedTemporaryFile(suffix=".obj", delete=False) as temp_file:
        mesh.export(temp_file.name)
        file_path = temp_file.name

    return file_path

theme = gr.themes.Soft(primary_hue="red", secondary_hue="red", font=['arial'])

with gr.Blocks(theme=theme) as demo:
    with gr.Column(elem_classes="header"):
        gr.Markdown("# MESA: Text-Driven Terrain Generation Using Latent Diffusion and Global Copernicus Data")
        gr.Markdown("### Paul Borne–Pons, Mikolaj Czerkawski, Rosalie Martin, Romain Rouffet")
        gr.Markdown('[[GitHub](https://github.com/PaulBorneP/MESA)] [[Model](https://huggingface.co/NewtNewt/MESA)] [[Dataset](https://huggingface.co/datasets/Major-TOM/Core-DEM)]')
        
    # Abstract Section
    with gr.Column(elem_classes="abstract"):
        gr.Markdown("MESA is a novel generative model based on latent denoising diffusion capable of generating 2.5D representations of terrain based on the text prompt conditioning supplied via natural language. The model produces two co-registered modalities of optical and depth maps.") # Replace with your abstract text
        gr.Markdown("This is a test version of the demo app. Please be aware that MESA supports primarily complex, mountainous terrains as opposed to flat land")
        gr.Markdown("The generated image is quite large, so for the full resolution (768) it might take a while to load the surface")

    with gr.Row():
        prompt_input = gr.Textbox(lines=2, placeholder="Enter a terrain description...")
        generate_button = gr.Button("Generate Terrain", variant="primary")

    model_output = gr.Model3D(
        camera_position=[90, 180, 512]
    )

    with gr.Accordion("Advanced Options", open=False) as advanced_options:
        num_inference_steps_slider = gr.Slider(minimum=10, maximum=1000, step=10, value=50, label="Inference Steps")
        guidance_scale_slider = gr.Slider(minimum=1.0, maximum=15.0, step=0.5, value=7.5, label="Guidance Scale")
        seed_number = gr.Number(value=6378, label="Seed")
        crop_size_slider = gr.Slider(minimum=128, maximum=768, step=64, value=512, label="Crop Size")
        prefix_textbox = gr.Textbox(label="Prompt Prefix", value="A Sentinel-2 image of ")

    generate_button.click(
        fn=generate_and_display,
        inputs=[prompt_input, num_inference_steps_slider, guidance_scale_slider, seed_number, crop_size_slider, prefix_textbox],
        outputs=model_output,
    )

if __name__ == "__main__":
    demo.launch(debug=True,
                share=True)