DINOV3 roof segmentation with google solar api

.gitignore

@@ -0,0 +1,15 @@
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+env/
+venv/
+.env
+*.egg-info/
+dist/
+build/
+.ipynb_checkpoints/
+*.log
+.DS_Store
+Thumbs.db

README.md

@@ -1,17 +1,36 @@
 ---
 title: Roof Segmentation DINOv3
-emoji: 🛰️
+emoji: 🏠
 colorFrom: blue
 colorTo: green
 sdk: gradio
-sdk_version: 4.44.0
+sdk_version: "4.44.0"
 app_file: app.py
 pinned: false
-license: other
+license: apache-2.0
 models:
   - facebook/dinov3-vitl16-pretrain-sat493m
 ---
 
 # Roof Segmentation with DINOv3 Satellite
 
-Segment roofs from satellite imagery using Meta's DINOv3 model pretrained on 493M satellite images.
+Extract roof polygons from satellite imagery using Meta's DINOv3 model pretrained on 493M satellite images.
+
+## Features
+
+- **Address Input** - Enter any US address
+- **Google Solar API** - Fetches high-resolution GeoTIFF imagery
+- **DINOv3 Segmentation** - State-of-the-art satellite image understanding
+- **GeoJSON Output** - Real-world coordinates for each roof polygon
+
+## Usage
+
+1. Enter a property address
+2. Click "Extract Roof Polygons"
+3. Identify which color clusters represent roofs
+4. Adjust cluster selection and re-run if needed
+5. Download the GeoJSON file
+
+## Requirements
+
+- Google Cloud API key with Solar API and Geocoding API enabled

app.py

@@ -1,16 +1,25 @@
 import gradio as gr
 import torch
-import torch.nn.functional as F
 import numpy as np
 from PIL import Image
 from transformers import AutoImageProcessor, AutoModel
 from sklearn.cluster import KMeans
+from sklearn.decomposition import PCA
+import cv2
+import json
+import requests
+import io
+import os
+import rasterio
+from rasterio.crs import CRS
 import warnings
 warnings.filterwarnings("ignore")
 
-# Model selection - ViT-L for satellite imagery
-MODEL_NAME = "facebook/dinov3-vitl16-pretrain-sat493m"
+# Load API key from environment (set as HF Space secret)
+GOOGLE_API_KEY = os.environ.get("GOOGLE_API_KEY", "")
 
+# DINOv3 Model - Satellite pretrained
+MODEL_NAME = "facebook/dinov3-vitl16-pretrain-sat493m"
 print(f"Loading {MODEL_NAME}...")
 device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 processor = AutoImageProcessor.from_pretrained(MODEL_NAME)
@@ -18,171 +27,392 @@ model = AutoModel.from_pretrained(MODEL_NAME).to(device)
 model.eval()
 print(f"Model loaded on {device}")
 
+
+def geocode_address(address, api_key):
+    """Convert address to lat/lng using Google Geocoding API."""
+    url = "https://maps.googleapis.com/maps/api/geocode/json"
+    params = {
+        "address": address,
+        "key": api_key
+    }
+    
+    response = requests.get(url, params=params)
+    data = response.json()
+    
+    if data["status"] != "OK":
+        raise ValueError(f"Geocoding failed: {data['status']}")
+    
+    location = data["results"][0]["geometry"]["location"]
+    formatted_address = data["results"][0]["formatted_address"]
+    
+    return location["lat"], location["lng"], formatted_address
+
+
+def fetch_geotiff(lat, lng, api_key, radius_meters=50):
+    """Fetch RGB GeoTIFF from Google Solar API Data Layers."""
+    
+    layers_url = "https://solar.googleapis.com/v1/dataLayers:get"
+    params = {
+        "location.latitude": lat,
+        "location.longitude": lng,
+        "radiusMeters": radius_meters,
+        "view": "FULL_LAYERS",
+        "requiredQuality": "HIGH",
+        "pixelSizeMeters": 0.25,
+        "key": api_key
+    }
+    
+    response = requests.get(layers_url, params=params)
+    
+    if response.status_code != 200:
+        params["requiredQuality"] = "MEDIUM"
+        response = requests.get(layers_url, params=params)
+    
+    if response.status_code != 200:
+        raise ValueError(f"Data Layers API error: {response.status_code} - {response.text}")
+    
+    layers = response.json()
+    
+    rgb_url = layers.get("rgbUrl")
+    if not rgb_url:
+        raise ValueError("No RGB imagery available for this location")
+    
+    rgb_response = requests.get(f"{rgb_url}&key={api_key}")
+    if rgb_response.status_code != 200:
+        raise ValueError(f"Failed to download GeoTIFF: {rgb_response.status_code}")
+    
+    return rgb_response.content, layers
+
+
+def parse_geotiff(geotiff_bytes):
+    """Parse GeoTIFF and extract image + bounds."""
+    
+    with rasterio.open(io.BytesIO(geotiff_bytes)) as src:
+        if src.count >= 3:
+            r = src.read(1)
+            g = src.read(2)
+            b = src.read(3)
+            img_array = np.stack([r, g, b], axis=-1)
+        else:
+            img_array = src.read(1)
+            img_array = np.stack([img_array] * 3, axis=-1)
+        
+        bounds = src.bounds
+        crs = src.crs
+        
+        if crs and crs != CRS.from_epsg(4326):
+            from rasterio.warp import transform_bounds
+            bounds = transform_bounds(crs, CRS.from_epsg(4326), *bounds)
+    
+    image = Image.fromarray(img_array.astype(np.uint8))
+    return image, bounds
+
+
 def extract_features(image):
     """Extract dense patch features from DINOv3."""
     inputs = processor(images=image, return_tensors="pt").to(device)
     
     with torch.inference_mode():
         outputs = model(**inputs)
-        # DINOv3: 1 CLS + 4 register tokens + N patch tokens
-        # Skip first 5 tokens (CLS + 4 registers)
+        # DINOv3: skip CLS + 4 register tokens
         patch_features = outputs.last_hidden_state[:, 5:, :]
     
     return patch_features
 
-def segment_roof(image, num_segments=5, selected_clusters="0"):
-    """
-    Segment roofs using DINOv3 satellite features + K-means.
+
+def pixel_to_geo(x, y, img_width, img_height, bounds):
+    """Convert pixel coordinates to geographic coordinates."""
+    west, south, east, north = bounds
+    
+    x_norm = x / img_width
+    y_norm = y / img_height
+    
+    lng = west + (east - west) * x_norm
+    lat = north - (north - south) * y_norm
+    
+    return [lng, lat]
+
+
+def mask_to_polygons(mask, bounds, img_width, img_height):
+    """Convert binary mask to GeoJSON polygons."""
+    features = []
     
-    Args:
-        image: Input satellite image
-        num_segments: Number of K-means clusters
-        selected_clusters: Comma-separated cluster indices to highlight as roof
-    """
-    if image is None:
-        return None, None, "Please upload an image"
+    contours, _ = cv2.findContours(
+        mask.astype(np.uint8), 
+        cv2.RETR_EXTERNAL,
+        cv2.CHAIN_APPROX_SIMPLE
+    )
     
-    # Convert to PIL if needed
-    if isinstance(image, np.ndarray):
-        image = Image.fromarray(image).convert("RGB")
+    for i, contour in enumerate(contours):
+        area = cv2.contourArea(contour)
+        if area < 100:
+            continue
+        
+        epsilon = 0.015 * cv2.arcLength(contour, True)
+        simplified = cv2.approxPolyDP(contour, epsilon, True)
+        
+        coords = []
+        for point in simplified:
+            px, py = point[0]
+            geo_coord = pixel_to_geo(px, py, img_width, img_height, bounds)
+            coords.append(geo_coord)
+        
+        if coords and coords[0] != coords[-1]:
+            coords.append(coords[0])
+        
+        if len(coords) >= 4:
+            west, south, east, north = bounds
+            meters_per_lng = 111320 * np.cos(np.radians((north + south) / 2))
+            meters_per_lat = 111320
+            pixel_width_m = (east - west) * meters_per_lng / img_width
+            pixel_height_m = (north - south) * meters_per_lat / img_height
+            area_sqm = area * pixel_width_m * pixel_height_m
+            
+            feature = {
+                "type": "Feature",
+                "properties": {
+                    "roof_id": i + 1,
+                    "area_sqm": round(area_sqm, 2),
+                    "area_sqft": round(area_sqm * 10.764, 2),
+                    "num_vertices": len(coords) - 1
+                },
+                "geometry": {
+                    "type": "Polygon",
+                    "coordinates": [coords]
+                }
+            }
+            features.append(feature)
     
-    original_size = image.size  # (W, H)
+    return features
+
+
+def segment_image(image, num_segments):
+    """Run DINOv3 segmentation on image."""
+    original_size = image.size
     
-    # Extract DINOv3 features
     features = extract_features(image)
     
-    # Calculate spatial dimensions
-    # DINOv3 uses patch_size=16
     num_patches = features.shape[1]
     h = w = int(np.sqrt(num_patches))
     
-    # Reshape for clustering
-    feat_np = features.squeeze(0).cpu().numpy()  # [num_patches, hidden_dim]
+    feat_np = features.squeeze(0).cpu().numpy()
     
-    # PCA for dimensionality reduction (helps clustering)
-    from sklearn.decomposition import PCA
     pca = PCA(n_components=64, random_state=42)
     feat_reduced = pca.fit_transform(feat_np)
     
-    # K-means clustering
     kmeans = KMeans(n_clusters=num_segments, random_state=42, n_init=10)
     cluster_labels = kmeans.fit_predict(feat_reduced)
     
-    # Reshape to spatial grid
     seg_map = cluster_labels.reshape(h, w)
-    
-    # Upscale to original image size
     seg_resized = np.array(
         Image.fromarray(seg_map.astype(np.uint8)).resize(
             original_size, resample=Image.NEAREST
         )
     )
     
-    # Color palette for visualization
-    colors = np.array([
-        [230, 25, 75],    # Red
-        [60, 180, 75],    # Green
-        [255, 225, 25],   # Yellow
-        [0, 130, 200],    # Blue
-        [245, 130, 48],   # Orange
-        [145, 30, 180],   # Purple
-        [70, 240, 240],   # Cyan
-        [240, 50, 230],   # Magenta
-        [210, 245, 60],   # Lime
-        [250, 190, 212],  # Pink
-    ])
-    
-    # Create colored segmentation
-    colored_seg = colors[seg_resized % len(colors)]
-    
-    # Parse selected clusters for roof mask
-    try:
-        roof_indices = [int(x.strip()) for x in selected_clusters.split(",") if x.strip()]
-    except:
-        roof_indices = [0]
+    return seg_resized
+
+
+def process_address(address, num_segments, selected_clusters, min_area, radius_meters, api_key_input):
+    """Main pipeline: address -> GeoJSON polygons."""
     
-    # Create binary roof mask
-    roof_mask = np.isin(seg_resized, roof_indices).astype(np.uint8) * 255
+    api_key = api_key_input.strip() if api_key_input.strip() else GOOGLE_API_KEY
     
-    # Create overlay visualization
-    orig_array = np.array(image).astype(np.float32)
-    overlay = orig_array * 0.4 + colored_seg.astype(np.float32) * 0.6
+    if not api_key:
+        return None, None, None, None, "❌ No API key provided. Enter your Google Solar API key."
     
-    # Highlight selected roof clusters
-    for idx in roof_indices:
-        mask = seg_resized == idx
-        overlay[mask] = orig_array[mask] * 0.3 + np.array([255, 0, 0]) * 0.7
+    try:
+        lat, lng, formatted_address = geocode_address(address, api_key)
+        status = f"📍 **{formatted_address}**\n\nCoordinates: {lat:.6f}, {lng:.6f}\n\n"
+    except Exception as e:
+        return None, None, None, None, f"❌ Geocoding failed: {str(e)}"
     
-    # Calculate cluster statistics
-    unique, counts = np.unique(seg_resized, return_counts=True)
-    total_pixels = seg_resized.size
-    stats = "**Cluster Statistics:**\n"
-    for u, c in sorted(zip(unique, counts), key=lambda x: -x[1]):
-        pct = (c / total_pixels) * 100
-        marker = " ← ROOF" if u in roof_indices else ""
-        stats += f"- Cluster {u}: {pct:.1f}%{marker}\n"
+    try:
+        status += "Fetching satellite imagery...\n"
+        geotiff_bytes, layers_info = fetch_geotiff(lat, lng, api_key, radius_meters)
+        image, bounds = parse_geotiff(geotiff_bytes)
+        img_width, img_height = image.size
+        status += f"Image size: {img_width}x{img_height}px\n\n"
+    except Exception as e:
+        return None, None, None, None, f"❌ Failed to fetch imagery: {str(e)}"
     
-    return overlay.astype(np.uint8), roof_mask, stats
+    try:
+        seg_resized = segment_image(image, num_segments)
+        
+        colors = np.array([
+            [230, 25, 75], [60, 180, 75], [255, 225, 25], [0, 130, 200],
+            [245, 130, 48], [145, 30, 180], [70, 240, 240], [240, 50, 230],
+            [210, 245, 60], [250, 190, 212], [128, 128, 0], [0, 128, 128]
+        ])
+        
+        colored_seg = colors[seg_resized % len(colors)]
+        
+        try:
+            roof_indices = [int(x.strip()) for x in selected_clusters.split(",") if x.strip()]
+        except:
+            roof_indices = [0]
+        
+        roof_mask = np.isin(seg_resized, roof_indices).astype(np.uint8) * 255
+        
+        kernel = np.ones((5, 5), np.uint8)
+        roof_mask = cv2.morphologyEx(roof_mask, cv2.MORPH_CLOSE, kernel)
+        roof_mask = cv2.morphologyEx(roof_mask, cv2.MORPH_OPEN, kernel)
+        
+        polygon_features = mask_to_polygons(roof_mask, bounds, img_width, img_height)
+        polygon_features = [f for f in polygon_features if f["properties"]["area_sqft"] >= min_area]
+        
+        geojson = {
+            "type": "FeatureCollection",
+            "properties": {
+                "source": "DINOv3 Roof Segmentation",
+                "address": formatted_address,
+                "center": {"lat": lat, "lng": lng},
+                "bounds": {
+                    "north": bounds[3], "south": bounds[1],
+                    "east": bounds[2], "west": bounds[0]
+                }
+            },
+            "features": polygon_features
+        }
+        
+        geojson_str = json.dumps(geojson, indent=2)
+        
+        orig_array = np.array(image).astype(np.float32)
+        overlay = orig_array * 0.4 + colored_seg.astype(np.float32) * 0.6
+        
+        for feature in polygon_features:
+            coords = feature["geometry"]["coordinates"][0]
+            pixel_coords = []
+            for lnglat in coords:
+                px = int((lnglat[0] - bounds[0]) / (bounds[2] - bounds[0]) * img_width)
+                py = int((bounds[3] - lnglat[1]) / (bounds[3] - bounds[1]) * img_height)
+                pixel_coords.append([px, py])
+            
+            pts = np.array(pixel_coords, dtype=np.int32)
+            cv2.polylines(overlay, [pts], True, (255, 255, 0), 3)
+        
+        for idx in roof_indices:
+            mask_highlight = seg_resized == idx
+            overlay[mask_highlight] = orig_array[mask_highlight] * 0.3 + np.array([255, 50, 50]) * 0.7
+        
+        total_sqft = sum(f["properties"]["area_sqft"] for f in polygon_features)
+        status += f"**Found {len(polygon_features)} roof polygon(s)**\n"
+        status += f"**Total roof area: {total_sqft:,.0f} sq ft**\n\n"
+        
+        for f in polygon_features:
+            props = f["properties"]
+            status += f"- Roof {props['roof_id']}: {props['area_sqft']:,.0f} sq ft\n"
+        
+        status += "\n**Cluster Distribution:**\n"
+        unique, counts = np.unique(seg_resized, return_counts=True)
+        total = seg_resized.size
+        for u, c in sorted(zip(unique, counts), key=lambda x: -x[1]):
+            pct = (c / total) * 100
+            marker = " ← ROOF" if u in roof_indices else ""
+            status += f"- Cluster {u}: {pct:.1f}%{marker}\n"
+        
+        return np.array(image), overlay.astype(np.uint8), roof_mask, geojson_str, status
+        
+    except Exception as e:
+        import traceback
+        return None, None, None, None, f"❌ Segmentation failed: {str(e)}\n\n{traceback.format_exc()}"
+
+
+def save_geojson(geojson_str):
+    """Save GeoJSON for download."""
+    if not geojson_str:
+        return None
+    filepath = "/tmp/roof_polygons.geojson"
+    with open(filepath, "w") as f:
+        f.write(geojson_str)
+    return filepath
+
 
 # Gradio Interface
-with gr.Blocks(title="Roof Segmentation - DINOv3 Satellite", theme=gr.themes.Soft()) as demo:
+with gr.Blocks(title="Roof Segmentation - Address to GeoJSON", theme=gr.themes.Soft()) as demo:
     gr.Markdown("""
-    # 🛰️ Roof Segmentation with DINOv3 (Satellite)
+    # 🏠 Address → Roof Polygons (GeoJSON)
     
-    Using Meta's **DINOv3 ViT-L** pretrained on **493M satellite images** at 0.6m resolution.
+    Enter an address, get roof segment polygons with real-world coordinates.
     
-    Upload a satellite/aerial image to detect and segment roof areas.
+    **Pipeline:** Address → Google Solar API (GeoTIFF) → DINOv3 Segmentation → GeoJSON
     """)
     
     with gr.Row():
         with gr.Column(scale=1):
-            input_image = gr.Image(type="pil", label="📸 Upload Satellite Image")
+            address_input = gr.Textbox(
+                label="📍 Property Address",
+                placeholder="123 Main St, Sacramento, CA",
+                lines=2
+            )
+            
+            with gr.Accordion("🔑 API Key", open=False):
+                api_key_input = gr.Textbox(
+                    label="Google Solar API Key",
+                    placeholder="Enter API key (or set GOOGLE_API_KEY secret)",
+                    type="password"
+                )
             
-            with gr.Accordion("⚙️ Segmentation Settings", open=True):
-                num_segments = gr.Slider(
-                    minimum=3, maximum=12, value=5, step=1,
-                    label="Number of Segments",
-                    info="More segments = finer detail"
+            with gr.Accordion("⚙️ Settings", open=True):
+                radius_meters = gr.Slider(
+                    25, 100, value=50, step=5,
+                    label="Image Radius (meters)",
+                    info="Area around the address to capture"
                 )
+                num_segments = gr.Slider(3, 12, value=6, step=1, label="Segments")
                 selected_clusters = gr.Textbox(
                     value="0", 
                     label="Roof Cluster(s)",
-                    info="Enter cluster numbers separated by commas (e.g., '0,2')",
-                    placeholder="0"
+                    placeholder="0,2,5"
+                )
+                min_area = gr.Slider(
+                    50, 2000, value=200, step=50,
+                    label="Min Roof Area (sq ft)"
                 )
             
-            segment_btn = gr.Button("🔍 Segment Roofs", variant="primary", size="lg")
+            process_btn = gr.Button("🔍 Extract Roof Polygons", variant="primary", size="lg")
         
         with gr.Column(scale=2):
             with gr.Row():
-                output_overlay = gr.Image(label="Segmentation Overlay")
-                output_mask = gr.Image(label="Roof Mask (Binary)")
+                original_img = gr.Image(label="Satellite Image")
+                overlay_img = gr.Image(label="Segmentation + Polygons")
             
-            cluster_stats = gr.Markdown(label="Cluster Info")
+            with gr.Row():
+                mask_img = gr.Image(label="Roof Mask")
+                status_output = gr.Markdown()
+            
+            with gr.Accordion("📄 GeoJSON Output", open=True):
+                geojson_output = gr.Code(language="json", lines=12)
+                download_btn = gr.Button("⬇️ Download GeoJSON")
+                download_file = gr.File(label="Download")
+    
+    process_btn.click(
+        fn=process_address,
+        inputs=[address_input, num_segments, selected_clusters, min_area, radius_meters, api_key_input],
+        outputs=[original_img, overlay_img, mask_img, geojson_output, status_output]
+    )
     
-    segment_btn.click(
-        fn=segment_roof,
-        inputs=[input_image, num_segments, selected_clusters],
-        outputs=[output_overlay, output_mask, cluster_stats]
+    download_btn.click(
+        fn=save_geojson,
+        inputs=[geojson_output],
+        outputs=[download_file]
     )
     
     gr.Markdown("""
     ---
     ### How to Use
-    1. Upload a satellite or aerial image of buildings
-    2. Click **Segment Roofs** to analyze
-    3. Look at the colored overlay - each color is a different segment
-    4. Find which cluster number(s) correspond to roofs (shown in stats)
-    5. Enter those numbers in **Roof Cluster(s)** and re-run
-    6. Download the binary mask for your workflow
-    
-    ### Tips
-    - **Roofs** often cluster together due to similar materials/colors
-    - Try **5-7 segments** for typical suburban imagery
-    - Multiple buildings? Select multiple clusters: `0,3,5`
+    1. Enter a US property address
+    2. Click **Extract Roof Polygons**
+    3. Review the segmentation - identify which cluster colors are roofs
+    4. Enter roof cluster numbers and re-run if needed
+    5. Download GeoJSON for your workflow
+    
+    ### Requirements
+    - Google Cloud project with **Solar API** and **Geocoding API** enabled
+    - API key with access to both APIs
     
     ---
-    *Powered by [DINOv3](https://github.com/facebookresearch/dinov3) pretrained on SAT-493M*
+    *Powered by DINOv3 (SAT-493M) + Google Solar API*
     """)
 
 demo.launch()

requirements.txt

@@ -3,4 +3,7 @@ transformers>=4.40.0
 gradio>=4.0.0
 Pillow
 numpy
-scikit-learn
+scikit-learn
+opencv-python-headless
+requests
+rasterio