app.py

import os
import numpy as np
import rasterio
from rasterio.transform import from_origin
from PIL import Image, ImageDraw
import trimesh
from tqdm import tqdm
from numba import njit, prange
from scipy.ndimage import binary_dilation, label
import torch
import gradio as gr
from transformers import Sam3Processor, Sam3Model
import warnings
import tempfile
import shutil

warnings.filterwarnings("ignore", category=RuntimeWarning)

# ---------------------------------------------------------------------
# Global SAM3 model
# ---------------------------------------------------------------------
device = "cuda" if torch.cuda.is_available() else "cpu"
print(f"Using device: {device}")

model = Sam3Model.from_pretrained("JobHarvest/sam3").to(device)
processor = Sam3Processor.from_pretrained("JobHarvest/sam3")

# =====================================================================
#  STEP 1: NUMBA RASTERISER
# =====================================================================

@njit(fastmath=True, cache=False)
def barycentric_coords(p, a, b, c):
    v0 = b - a
    v1 = c - a
    v2 = p - a

    d00 = v0[0]*v0[0] + v0[1]*v0[1]
    d01 = v0[0]*v1[0] + v0[1]*v1[1]
    d11 = v1[0]*v1[0] + v1[1]*v1[1]
    d20 = v2[0]*v0[0] + v2[1]*v0[1]
    d21 = v2[0]*v1[0] + v2[1]*v1[1]

    denom = d00 * d11 - d01 * d01
    if abs(denom) < 1e-12:
        return np.array([-1.0, -1.0, -1.0], dtype=np.float64)

    v = (d11 * d20 - d01 * d21) / denom
    w = (d00 * d21 - d01 * d20) / denom
    u = 1.0 - v - w
    return np.array([u, v, w], dtype=np.float64)


@njit(fastmath=True, cache=False, parallel=True)
def rasterize_triangles(
    ortho, depth_buffer, texture, faces, vertices, uv,
    min_x, max_y, resolution, width, height
):
    tex_h, tex_w = texture.shape[:2]

    for f_idx in prange(len(faces)):
        face = faces[f_idx]
        verts = vertices[face]
        uv_tri = uv[face]

        pts = verts[:, :2]

        px = ((pts[:, 0] - min_x) / resolution).astype(np.int32)
        py = ((max_y - pts[:, 1]) / resolution).astype(np.int32)

        min_px = max(px.min(), 0)
        max_px = min(px.max(), width - 1)
        min_py = max(py.min(), 0)
        max_py = min(py.max(), height - 1)

        if min_px >= max_px or min_py >= max_py:
            continue

        a = np.array([float(px[0]), float(py[0])], dtype=np.float64)
        b = np.array([float(px[1]), float(py[1])], dtype=np.float64)
        c = np.array([float(px[2]), float(py[2])], dtype=np.float64)

        z_vals = verts[:, 2]

        for iy in range(min_py, max_py + 1):
            for ix in range(min_px, max_px + 1):
                p = np.array([float(ix), float(iy)], dtype=np.float64)

                bc = barycentric_coords(p, a, b, c)

                if bc[0] >= -1e-6 and bc[1] >= -1e-6 and bc[2] >= -1e-6:
                    z = bc[0]*z_vals[0] + bc[1]*z_vals[1] + bc[2]*z_vals[2]

                    if z > depth_buffer[iy, ix]:
                        depth_buffer[iy, ix] = z

                        uv_interp = (bc[0] * uv_tri[0] +
                                     bc[1] * uv_tri[1] +
                                     bc[2] * uv_tri[2])

                        u = max(0.0, min(1.0, uv_interp[0]))
                        v = max(0.0, min(1.0, uv_interp[1]))

                        tx = int(u * (tex_w - 1))
                        ty = int((1.0 - v) * (tex_h - 1))

                        ortho[iy, ix, 0] = texture[ty, tx, 0]
                        ortho[iy, ix, 1] = texture[ty, tx, 1]
                        ortho[iy, ix, 2] = texture[ty, tx, 2]


def load_textured_mesh(obj_path):
    mesh = trimesh.load(obj_path, force='mesh', process=False)
    if isinstance(mesh, trimesh.Scene):
        mesh = trimesh.util.concatenate(
            [g for g in mesh.geometry.values() if isinstance(g, trimesh.Trimesh)]
        )
    print(f"Vertices : {len(mesh.vertices):,}")
    print(f"Faces    : {len(mesh.faces):,}")
    if mesh.visual.uv is None:
        raise ValueError("Mesh has no UV coordinates")
    return mesh


def load_texture_image(mesh):
    if hasattr(mesh.visual.material, "image"):
        texture = mesh.visual.material.image
    else:
        raise ValueError("Texture image not found")
    if isinstance(texture, Image.Image):
        texture = np.array(texture.convert("RGB"))
    return texture.astype(np.uint8)


def generate_orthophoto_numba(obj_path, output_tif, resolution=0.1):
    print("Loading mesh...")
    mesh = load_textured_mesh(obj_path)
    texture = load_texture_image(mesh)

    vertices = mesh.vertices.astype(np.float64)
    faces = mesh.faces.astype(np.int32)
    uv = mesh.visual.uv.astype(np.float64)

    min_x = vertices[:, 0].min()
    max_x = vertices[:, 0].max()
    min_y = vertices[:, 1].min()
    max_y = vertices[:, 1].max()

    width = int(np.ceil((max_x - min_x) / resolution))
    height = int(np.ceil((max_y - min_y) / resolution))

    print(f"Orthophoto size: {width:,} x {height:,} pixels")

    ortho = np.zeros((height, width, 3), dtype=np.uint8)
    depth_buffer = np.full((height, width), -np.inf, dtype=np.float64)

    print("Rasterizing mesh...")
    rasterize_triangles(
        ortho, depth_buffer, texture, faces, vertices, uv,
        min_x, max_y, resolution, width, height
    )

    transform = from_origin(min_x, max_y, resolution, resolution)
    with rasterio.open(
        output_tif, "w",
        driver="GTiff",
        width=width, height=height,
        count=3, dtype=np.uint8,
        transform=transform,
        compress="LZW"
    ) as dst:
        dst.write(ortho.transpose(2, 0, 1))

    meta = (min_x, max_x, min_y, max_y, resolution)
    return output_tif, ortho, meta, mesh


# =====================================================================
#  STEP 2: SAM3 SEGMENTATION WITH YOUR EXACT TILING LOGIC
# =====================================================================

def tile_image(image, grid=(2, 2)):
    w, h = image.size
    tile_w, tile_h = w // grid[0], h // grid[1]
    tiles = []
    for i in range(grid[0]):
        for j in range(grid[1]):
            left = i * tile_w
            top = j * tile_h
            tile = image.crop((left, top, left + tile_w, top + tile_h))
            tiles.append((tile, (left, top)))
    return tiles, (w, h)

def segment_tile(tile, prompt="car"):
    inputs = processor(images=tile, text=prompt, return_tensors="pt").to(device)
    with torch.no_grad():
        outputs = model(**inputs)
    results = processor.post_process_instance_segmentation(
        outputs,
        threshold=0.35,
        mask_threshold=0.5,
        target_sizes=[(tile.height, tile.width)]
    )[0]
    return results['masks']

def combine_masks(tile_masks_list, offsets, full_size):
    h, w = full_size[1], full_size[0]
    combined = np.zeros((h, w), dtype=np.int32)
    next_id = 1

    for tile_idx, masks in enumerate(tile_masks_list):
        xoff, yoff = offsets[tile_idx]
        for mask in masks:
            mask_np = mask.squeeze().cpu().numpy().astype(bool)
            th, tw = mask_np.shape
            combined[yoff:yoff+th, xoff:xoff+tw][mask_np] = next_id
            next_id += 1
    return combined


def segment_orthophoto_tiled(image_path, prompt="car", grid=(2, 2)):
    image = Image.open(image_path).convert("RGB")
    tiles, (full_w, full_h) = tile_image(image, grid=grid)
    offsets = [off for _, off in tiles]
    tile_images = [tile for tile, _ in tiles]

    all_tile_masks = []
    for tile in tile_images:
        masks = segment_tile(tile, prompt=prompt)
        all_tile_masks.append(masks)

    combined_label = combine_masks(all_tile_masks, offsets, (full_w, full_h))
    binary_mask = (combined_label > 0).astype(np.uint8) * 255
    return binary_mask


# =====================================================================
#  HELPER: Create overlay composite for ImageEditor
# =====================================================================

def create_mask_overlay(ortho_pil, mask_pil, color=(255, 0, 0), alpha=0.4):
    """
    Overlay the binary mask on the orthophoto with a semi‑transparent color.
    Returns a composite PIL image.
    """
    # Ensure both are RGB
    if ortho_pil.mode != 'RGB':
        ortho_pil = ortho_pil.convert('RGB')
    if mask_pil.mode != 'L':
        mask_pil = mask_pil.convert('L')
    # Create a red overlay where mask is white
    overlay = Image.new('RGBA', ortho_pil.size, (0, 0, 0, 0))
    draw = ImageDraw.Draw(overlay)
    # Convert mask to numpy for faster pixel operations
    mask_np = np.array(mask_pil)
    # Make red channel = 255, alpha = alpha where mask > 128
    red_overlay = np.zeros((*mask_np.shape, 4), dtype=np.uint8)
    red_overlay[mask_np > 128] = [color[0], color[1], color[2], int(alpha * 255)]
    overlay = Image.fromarray(red_overlay, 'RGBA')
    # Composite on orthophoto
    composite = Image.alpha_composite(ortho_pil.convert('RGBA'), overlay).convert('RGB')
    return composite


# =====================================================================
#  STEP 3: FLATTEN
# =====================================================================

def flatten_mesh_by_mask(mesh, mask, gsd, origin_x, origin_y,
                         flatten_buffer_m=1.0,
                         max_edge_length=30.0,
                         height_tolerance=1.0,
                         min_component_area=5):
    vertices = mesh.vertices.copy()
    faces = mesh.faces.copy()
    n_verts = len(vertices)
    labeled_mask, num_labels = label(mask)
    print(f"Found {num_labels} components.")

    modified_vertices = np.zeros(n_verts, dtype=bool)

    for label_id in range(1, num_labels + 1):
        comp_mask = (labeled_mask == label_id).astype(np.uint8)
        if np.sum(comp_mask) < min_component_area:
            continue

        h, w = mask.shape
        col = (vertices[:, 0] - origin_x) / gsd
        row = h - (vertices[:, 1] - origin_y) / gsd
        col_int = np.clip(np.round(col).astype(np.int64), 0, w - 1)
        row_int = np.clip(np.round(row).astype(np.int64), 0, h - 1)
        valid = (col >= 0) & (col < w) & (row >= 0) & (row < h)
        inside = np.zeros(n_verts, dtype=bool)
        inside[valid] = comp_mask[row_int[valid], col_int[valid]] > 0
        z_vals = vertices[inside, 2]
        if len(z_vals) == 0:
            continue
        q1, q3 = np.percentile(z_vals, [25, 75])
        iqr = q3 - q1
        valid_z = z_vals[(z_vals >= q1 - 1.5*iqr) & (z_vals <= q3 + 1.5*iqr)]
        if len(valid_z) == 0:
            valid_z = z_vals
        target_z = np.percentile(valid_z, 5)

        r_pix = int(np.ceil(flatten_buffer_m / gsd))
        struct = np.ones((2*r_pix+1, 2*r_pix+1), dtype=bool)
        expanded = binary_dilation(comp_mask, structure=struct).astype(np.uint8)

        in_expanded = np.zeros(n_verts, dtype=bool)
        in_expanded[valid] = expanded[row_int[valid], col_int[valid]] > 0

        v0 = vertices[faces[:,0]]
        v1 = vertices[faces[:,1]]
        v2 = vertices[faces[:,2]]
        edge1 = np.linalg.norm(v1 - v0, axis=1)
        edge2 = np.linalg.norm(v2 - v1, axis=1)
        edge3 = np.linalg.norm(v0 - v2, axis=1)
        max_edges = np.maximum(np.maximum(edge1, edge2), edge3)
        long_faces = max_edges > max_edge_length

        face_touching = np.any(in_expanded[faces], axis=1)
        face_touching = face_touching & ~long_faces

        candidate_vertices = np.unique(faces[face_touching].ravel())
        vert_z = vertices[candidate_vertices, 2]
        z_mask = (vert_z >= target_z - height_tolerance) & (vert_z <= target_z + height_tolerance)
        vertices_to_flatten = candidate_vertices[z_mask]

        if len(vertices_to_flatten) > 0:
            vertices[vertices_to_flatten, 2] = target_z
            modified_vertices[vertices_to_flatten] = True
            print(f"Component {label_id}: flattened {len(vertices_to_flatten)} vertices.")

    new_mesh = trimesh.Trimesh(vertices=vertices, faces=faces, process=False)
    modified_indices = np.where(modified_vertices)[0]
    print(f"Total modified vertices: {len(modified_indices)}/{n_verts}")
    return new_mesh, modified_indices


# =====================================================================
#  Load existing TIF
# =====================================================================

def load_existing_tif(tif_file):
    if tif_file is None:
        return None, None, None

    if hasattr(tif_file, 'name'):
        tif_path = tif_file.name
    else:
        tif_path = str(tif_file)

    if not os.path.exists(tif_path):
        return None, None, f"File not found: {tif_path}"

    try:
        with rasterio.open(tif_path) as src:
            if src.count >= 3:
                r = src.read(1)
                g = src.read(2)
                b = src.read(3)
                ortho_arr = np.stack([r, g, b], axis=-1)
            else:
                ortho_arr = np.stack([src.read(1)] * 3, axis=-1)

            if ortho_arr.dtype != np.uint8:
                ortho_arr = (ortho_arr / ortho_arr.max() * 255).astype(np.uint8)

            transform = src.transform
            min_x = transform.c
            max_y = transform.f
            resolution = abs(transform.a)
            meta = (min_x, min_x + src.width * resolution, max_y - src.height * resolution, max_y, resolution)
            pil_img = Image.fromarray(ortho_arr)
            return tif_path, pil_img, meta

    except Exception as e:
        return None, None, f"Error reading TIF: {e}"


# =====================================================================
#  GRADIO UI
# =====================================================================

def build_ui():
    with gr.Blocks(title="Orthophoto & Flattening Pipeline") as demo:
        gr.Markdown("## Orthophoto Generation + SAM3 Segmentation + Mesh Flattening")
        gr.Markdown("You can either generate an orthophoto from OBJ, or load an existing GeoTIFF.")

        with gr.Row():
            obj_input = gr.Textbox(
                label="OBJ file path (required for flattening)",
                value="D:/otto/obj/11-SE-2C-20/Tile_+264_+112.obj"
            )
            out_dir = gr.Textbox(
                label="Output directory",
                value="D:/otto/obj/11-SE-2C-20/"
            )
            resolution = gr.Number(value=0.1, label="GSD (m/pixel) for generation", step=0.01)

        # States
        ortho_path_state = gr.State()
        ortho_image_state = gr.State()
        mesh_state = gr.State()
        meta_state = gr.State()

        # Step 1
        with gr.Row():
            gen_ortho_btn = gr.Button("1a. Generate Orthophoto from OBJ")
            load_tif_btn = gr.Button("1b. Load existing GeoTIFF")
        with gr.Row():
            tif_upload = gr.File(label="Upload GeoTIFF", file_types=[".tif", ".tiff"])
        ortho_output = gr.Image(label="Orthophoto", type="pil")
        ortho_status = gr.Textbox(label="Status")

        # Step 2
        with gr.Row():
            prompt_input = gr.Textbox(value="car", label="Prompt for SAM3")
            grid_choice = gr.Dropdown(
                choices=["1x1 (no tiling)", "2x2", "3x3", "4x4", "5x5"],
                value="2x2",
                label="Tiling grid (larger = less memory)"
            )
            segment_btn = gr.Button("2. Segment & Edit Mask")
        with gr.Row():
            mask_editor = gr.ImageEditor(
                label="Mask Editor (orthophoto background with mask overlay)",
                type="pil",
                brush=dict(default_size=20, colors=["#FFFFFF", "#000000"]),
                eraser=dict(default_size=20)
            )
            use_mask_check = gr.Checkbox(label="Use this mask for flattening", value=True)
        mask_status = gr.Textbox(label="Mask Status")

        # Step 3
        with gr.Row():
            flatten_btn = gr.Button("3. Flatten Mesh")
        flatten_output = gr.File(label="Download flattened OBJ")
        flatten_status = gr.Textbox(label="Flatten Status")

        # ---------- Callbacks ----------
        def step1_gen(obj_path, out_dir, res):
            if not os.path.exists(obj_path):
                return None, "OBJ file not found.", None, None, None, None
            out_tif = os.path.join(out_dir, "orthophoto.tif")
            try:
                tif_path, ortho_arr, meta, mesh = generate_orthophoto_numba(obj_path, out_tif, res)
                ortho_pil = Image.fromarray(ortho_arr)
                return ortho_pil, "Orthophoto generated.", mesh, tif_path, meta, ortho_pil
            except Exception as e:
                return None, f"Error: {e}", None, None, None, None

        gen_ortho_btn.click(
            step1_gen,
            inputs=[obj_input, out_dir, resolution],
            outputs=[ortho_output, ortho_status, mesh_state, ortho_path_state, meta_state, ortho_image_state]
        )

        def step1_load(tif_file):
            if tif_file is None:
                return None, "No file selected.", None, None, None, None
            tif_path, pil_img, meta = load_existing_tif(tif_file)
            if pil_img is None:
                return None, str(meta), None, None, None, None
            return pil_img, f"Loaded TIF: {tif_path}", None, tif_path, meta, pil_img

        load_tif_btn.click(
            step1_load,
            inputs=[tif_upload],
            outputs=[ortho_output, ortho_status, mesh_state, ortho_path_state, meta_state, ortho_image_state]
        )

        def step2(ortho_path, prompt, grid_choice, ortho_pil):
            if ortho_path is None or not os.path.exists(ortho_path):
                return None, "Please generate or load an orthophoto first."
            if ortho_pil is None:
                return None, "Orthophoto image not available in state."
            try:
                if grid_choice == "1x1 (no tiling)":
                    grid = (1, 1)
                else:
                    size = int(grid_choice.split("x")[0])
                    grid = (size, size)
                mask = segment_orthophoto_tiled(ortho_path, prompt, grid)
                mask_pil = Image.fromarray(mask)
                # Create composite overlay (mask in red, semi-transparent)
                composite = create_mask_overlay(ortho_pil, mask_pil, color=(255, 0, 0), alpha=0.4)
                # Return the EditorValue dict
                editor_value = {
                    "background": ortho_pil,
                    "layers": [mask_pil],
                    "composite": composite
                }
                return editor_value, f"Mask generated with {grid[0]}x{grid[1]} tiling. Overlay shows mask on orthophoto."
            except Exception as e:
                return None, f"Segmentation failed: {e}"

        segment_btn.click(
            step2,
            inputs=[ortho_path_state, prompt_input, grid_choice, ortho_image_state],
            outputs=[mask_editor, mask_status]
        )

        def step3(obj_path, out_dir, editor_output, use_mask, meta):
            if not use_mask:
                return None, "Mask not used (skip flattening)."
            if editor_output is None:
                return None, "No mask available. Segment first."

            # Extract the final mask image from editor output
            mask_pil = None
            if isinstance(editor_output, dict):
                mask_pil = editor_output.get('composite')
                if mask_pil is None:
                    # fallback: try layers
                    layers = editor_output.get('layers')
                    if layers and len(layers) > 0:
                        mask_pil = layers[0]
                    else:
                        for val in editor_output.values():
                            if isinstance(val, Image.Image):
                                mask_pil = val
                                break
            elif isinstance(editor_output, Image.Image):
                mask_pil = editor_output

            if mask_pil is None:
                return None, "Could not extract mask from editor."

            # Convert to binary
            mask = np.array(mask_pil.convert("L"))
            mask = (mask > 128).astype(np.uint8)
            if np.sum(mask) == 0:
                return None, "Mask is empty. Please draw some regions."

            if meta is None:
                return None, "Metadata missing. Generate or load orthophoto first."
            min_x, max_x, min_y, max_y, gsd = meta
            origin_x = min_x
            origin_y = min_y

            if not os.path.exists(obj_path):
                return None, f"OBJ file not found: {obj_path}"
            mesh = load_textured_mesh(obj_path)
            flat_mesh, modified = flatten_mesh_by_mask(
                mesh, mask, gsd, origin_x, origin_y,
                flatten_buffer_m=1.0,
                max_edge_length=30.0,
                height_tolerance=1.0,
                min_component_area=5
            )
            out_obj = os.path.join(out_dir, "flattened.obj")
            flat_mesh.export(out_obj)

            # Temporary copy for Gradio
            tmp_obj = tempfile.NamedTemporaryFile(suffix=".obj", delete=False)
            tmp_obj.close()
            shutil.copy(out_obj, tmp_obj.name)

            return tmp_obj.name, f"Flattening done. {len(modified)} vertices modified. (Saved to {out_obj})"

        flatten_btn.click(
            step3,
            inputs=[obj_input, out_dir, mask_editor, use_mask_check, meta_state],
            outputs=[flatten_output, flatten_status]
        )

    return demo


if __name__ == "__main__":
    demo = build_ui()
    demo.launch(share=False)