feat: two-step image gen, Dockerfile builds floorplan whl at deploy time

.gitignore

@@ -1 +1,25 @@
 .worktrees/
+.env
+
+# Python
+__pycache__/
+*.pyc
+*.pyo
+.venv/
+*.egg-info/
+
+# Node / frontend
+node_modules/
+
+# macOS
+.DS_Store
+
+# Built artifacts (built by Dockerfile at deploy time)
+floorplan/dist/
+
+# Dev/test files
+test_gemini.py
+
+# Docs artifacts
+docs/.DS_Store
+docs/superpowers/.DS_Store

Dockerfile

@@ -0,0 +1,28 @@
+FROM python:3.12-slim
+
+# Install Node.js for building the frontend
+RUN apt-get update && apt-get install -y curl && \
+    curl -fsSL https://deb.nodesource.com/setup_20.x | bash - && \
+    apt-get install -y nodejs && \
+    apt-get clean && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+# Copy floorplan source first and build the whl
+COPY floorplan/ ./floorplan/
+RUN pip install gradio hatchling hatch-requirements-txt hatch-fancy-pypi-readme build && \
+    cd floorplan && \
+    pip install gradio-cli && \
+    gradio cc build --no-generate-docs && \
+    pip install dist/gradio_floorplan-0.0.1-py3-none-any.whl
+
+# Install remaining app dependencies
+COPY requirements.txt ./
+RUN pip install google-genai python-dotenv pillow pydantic
+
+# Copy app
+COPY app.py ./
+
+EXPOSE 7860
+
+CMD ["python", "app.py"]

app.py

@@ -1,43 +1,239 @@
+import os
+import json
+from io import BytesIO
+from typing import List
+from dotenv import load_dotenv
+
 import gradio as gr
 from gradio_floorplan import FloorPlan
+from PIL import Image
+from pydantic import BaseModel, Field
+from google import genai
+
+from google.genai import types
+
+load_dotenv()
+client = genai.Client()
+
+print("--- LA MAISON APP LOADED (VERSION 1.4 - OPTIMIZED IMAGE GEN & UI) ---")
 
 DEFAULT_FLOOR_PLAN = {
-    "corners": [[50, 50], [550, 50], [550, 450], [50, 450]],
-    "furnitures": [
-        {
-            "object": "Sofa",
-            "localisation": [150, 100, 250, 300],
-            "description": "3-seat sofa",
-        },
-        {
-            "object": "Table",
-            "localisation": [300, 200, 380, 400],
-            "description": "Coffee table",
-        },
-    ],
+    "corners": [],
+    "furnitures": [],
 }
 
+class FurnitureItem(BaseModel):
+    object: str
+    localisation: List[int] = Field(description="[ymin, xmin, ymax, xmax] coordinates")
+    description: str
+
+class RoomLayout(BaseModel):
+    corners: List[List[int]] = Field(description="List of [x, y] coordinates for room corners")
+    furnitures: List[FurnitureItem]
+
+def process_room_image(image_numpy):
+    if image_numpy is None:
+        return DEFAULT_FLOOR_PLAN, DEFAULT_FLOOR_PLAN
+    
+    pil_image = Image.fromarray(image_numpy)
+    
+    # Step 1: Generate Floor Plan image with Nano Banana 2 (Optimized)
+    generated_image = None
+    gr.Info("Step 1: Generating 2D floor plan image...")
+    print("\n--- Starting Gemini 3.1 Flash Image Preview (Nano Banana 2) - Optimized ---")
+    
+    generate_content_config = types.GenerateContentConfig(
+        thinking_config=types.ThinkingConfig(
+            thinking_level="MINIMAL",
+        ),
+        image_config = types.ImageConfig(
+            image_size="1K",
+        ),
+        response_modalities=[
+            "IMAGE",
+        ],
+    )
+
+    try:
+        response_stream = client.models.generate_content_stream(
+            model='models/gemini-3.1-flash-image-preview',
+            contents=[pil_image, "Generate a clean, 2D top-down floor plan image representing this exact room. OUTPUT ONLY THE IMAGE."],
+            config=generate_content_config
+        )
+        for chunk in response_stream:
+            if chunk.parts:
+                for part in chunk.parts:
+                    if part.inline_data:
+                        print("[Image Data] Received inline image data!")
+                        generated_image = Image.open(BytesIO(part.inline_data.data))
+                        break
+            
+        print("--- Finished generating floor plan image ---\n")
+        gr.Info("Step 1 Complete: Floor plan image generated!")
+    except Exception as e:
+        gr.Warning(f"Error generating image: {e}")
+        print(f"Error generating image: {e}")
+    
+    # Step 2: Extract coordinates
+    layout_json = DEFAULT_FLOOR_PLAN
+    image_to_parse = generated_image if generated_image else pil_image
+    
+    gr.Info("Step 2: Analyzing top plan image to extract coordinates...")
+    
+    prompted_model = "gemini-3-flash-preview" 
+
+    try:
+        response_json = client.models.generate_content(
+            model=prompted_model,
+            contents=[
+                image_to_parse, 
+                "OUTPUT FORMAT INSTRUCTION: Return ONLY a valid JSON string. DO NOT provide any reasoning, markdown formatting, or conversational text.\n"
+                "Analyze this floor plan (or room image) and output the exact coordinates "
+                "for the room corners formatted as [[x, y], ...] and a list of furnitures with "
+                "their object name, description, and bounding box [ymin, xmin, ymax, xmax]."
+            ],
+            config={
+                'response_mime_type': 'application/json',
+                'response_schema': RoomLayout
+            }
+        )
+        if response_json.text:
+            layout_json = json.loads(response_json.text)
+            gr.Info("Step 2 Complete: Coordinates extracted successfully!")
+            print(layout_json)
+    except Exception as e:
+        gr.Warning(f"Error parsing layout JSON: {e}")
+        print(f"Error parsing layout JSON: {e}")
+        
+    # Return: floor_plan value, initial_layout_state, topview_state, topview_before display
+    return layout_json, layout_json, generated_image, generated_image
+
+def _generate_image(contents, label="image"):
+    """Helper: call Gemini image generation and return a PIL Image or None."""
+    config = types.GenerateContentConfig(
+        thinking_config=types.ThinkingConfig(thinking_level="MINIMAL"),
+        image_config=types.ImageConfig(image_size="1K"),
+        response_modalities=["IMAGE"],
+    )
+    try:
+        response_stream = client.models.generate_content_stream(
+            model='models/gemini-3.1-flash-image-preview',
+            contents=contents,
+            config=config,
+        )
+        for chunk in response_stream:
+            if chunk.parts:
+                for part in chunk.parts:
+                    if part.inline_data:
+                        print(f"[Image Data] Received {label}")
+                        return Image.open(BytesIO(part.inline_data.data))
+    except Exception as e:
+        gr.Warning(f"Error generating {label}: {e}")
+        print(f"Error generating {label}: {e}")
+    return None
+
+
+def generate_final_image(original_image_numpy, current_layout_json, initial_layout_json, original_topview):
+    if original_image_numpy is None:
+        gr.Warning("Please upload an image first.")
+        return None, None
 
-def on_furniture_moved(value):
-    return value
+    pil_image = Image.fromarray(original_image_numpy)
 
+    # Compute changed furnitures
+    old_furnitures = {f["object"]: f for f in (initial_layout_json.get("furnitures") or [])}
+    new_furnitures = {f["object"]: f for f in (current_layout_json.get("furnitures") or [])}
+    moved = [
+        {"object": name, "from": old_furnitures[name]["localisation"], "to": new_furnitures[name]["localisation"]}
+        for name in new_furnitures
+        if name in old_furnitures and old_furnitures[name]["localisation"] != new_furnitures[name]["localisation"]
+    ]
+    added = [f for name, f in new_furnitures.items() if name not in old_furnitures]
+    removed = [f for name, f in old_furnitures.items() if name not in new_furnitures]
+    changes = {"moved": moved, "added": added, "removed": removed}
+    changes_str = json.dumps(changes, indent=2)
+    new_layout_str = json.dumps(current_layout_json, indent=2)
+    print(f"Furniture changes: {changes_str}")
+
+    # Step A: original top-view + new layout → new top-view
+    gr.Info("Step A: Generating updated top-view floor plan image...")
+    print("\n--- Step A: new top-view from floor plan ---")
+    step_a_input = original_topview if original_topview is not None else pil_image
+    prompt_a = f"""This is a 2D top-down floor plan image of a room.
+Redraw it as a clean 2D top-down floor plan image applying the following furniture changes.
+Coordinates are bounding boxes [ymin, xmin, ymax, xmax].
+
+Changes to apply:
+{changes_str}
+
+Full new target layout for reference:
+{new_layout_str}
+
+OUTPUT ONLY THE IMAGE. Keep the same room boundaries and style."""
+
+    new_topview = _generate_image([step_a_input, prompt_a], label="new top-view")
+    if new_topview is None:
+        gr.Warning("Step A failed: could not generate updated top-view.")
+        return None, None
+
+    gr.Info("Step A complete. Generating final photorealistic image...")
+
+    # Step B: new top-view + original photo → final photorealistic image
+    print("\n--- Step B: photorealistic synthesis ---")
+    prompt_b = """You are given two images:
+1. A 2D top-down floor plan showing the NEW furniture layout.
+2. The original room photo.
+
+Generate a high-quality photorealistic image of the room from the EXACT SAME camera angle as the original photo, but with the furniture repositioned to match the new floor plan.
+
+STRICT INSTRUCTIONS:
+1. Maintain the exact camera point-of-view, lighting, colors, and architectural features of the original photo.
+2. Place furniture according to the new floor plan layout.
+3. The room must look realistic and clean."""
+
+    final_image = _generate_image([new_topview, pil_image, prompt_b], label="final photorealistic")
+
+    if final_image is None:
+        gr.Warning("Step B failed: could not generate final image.")
+        return new_topview, None
+
+    gr.Info("Final image generated successfully!")
+    return new_topview, final_image
 
 with gr.Blocks() as demo:
     gr.Markdown("# LaMaison\nRearrange your spaces using visual planning.")
+    
+    # State to store the initial layout snapshot and the original top-view image
+    initial_layout_state = gr.State(DEFAULT_FLOOR_PLAN)
+    topview_state = gr.State(None)
 
-    with gr.Tab("Floor Plan"):
-        floor_plan = FloorPlan(
-            value=DEFAULT_FLOOR_PLAN,
-            label="Floor Plan",
-            interactive=True,
-        )
-        positions_output = gr.JSON(label="Current furniture positions")
-        floor_plan.change(on_furniture_moved, inputs=floor_plan, outputs=positions_output)
+    with gr.Row():
+        with gr.Column(scale=1):
+            image_input = gr.Image(label="Upload a room image", type="numpy")
+            floor_plan = FloorPlan(
+                value=DEFAULT_FLOOR_PLAN,
+                label="Floor Plan",
+                interactive=True,
+            )
+            generate_button = gr.Button("Generate New Image based on Floor plan", variant="primary")
+        with gr.Column(scale=1):
+            topview_before = gr.Image(label="Top View — Original", interactive=False)
+            topview_after = gr.Image(label="Top View — New Layout (Step A)", interactive=False)
+            image_output = gr.Image(label="Final Photorealistic Output (Step B)", interactive=False)
+
+    # Wire it: input -> [floor_plan, initial_layout_state, topview_state, topview_before]
+    image_input.change(
+        process_room_image,
+        inputs=image_input,
+        outputs=[floor_plan, initial_layout_state, topview_state, topview_before]
+    )
 
-    with gr.Tab("Image"):
-        image_input = gr.Image(label="Upload a room image", type="numpy")
-        image_output = gr.Image(label="Output")
-        image_input.change(lambda x: x, inputs=image_input, outputs=image_output)
+    # Phase 2: User clicks button -> generates new top-view then final photorealistic image
+    generate_button.click(
+        generate_final_image,
+        inputs=[image_input, floor_plan, initial_layout_state, topview_state],
+        outputs=[topview_after, image_output]
+    )
 
 if __name__ == "__main__":
-    demo.launch()
+    demo.launch(server_name="0.0.0.0", server_port=7860)

floorplan/.gitignore

@@ -1,5 +1,5 @@
 .eggs/
-dist/
+dist/*.tar.gz
 *.pyc
 __pycache__/
 *.py[cod]

floorplan/README.md

@@ -1,10 +1,262 @@
 
-# gradio_floorplan
-A Custom Gradio component.
+# `gradio_floorplan`
+<img alt="Static Badge" src="https://img.shields.io/badge/version%20-%200.0.1%20-%20orange">  
 
-## Example usage
+A Gradio custom component for interactive SVG floor plan editing
+
+## Installation
+
+```bash
+pip install gradio_floorplan
+```
+
+## Usage
 
 ```python
 import gradio as gr
 from gradio_floorplan import FloorPlan
+
+DEFAULT_VALUE = {
+    "corners": [[50, 50], [550, 50], [550, 450], [50, 450]],
+    "furnitures": [
+        {
+            "object": "Sofa",
+            "localisation": [150, 100, 250, 300],
+            "description": "3-seat sofa",
+        },
+        {
+            "object": "Table",
+            "localisation": [300, 200, 380, 400],
+            "description": "Coffee table",
+        },
+    ],
+}
+
+
+def on_furniture_moved(value):
+    """Receives updated floor plan after user moves a furniture item."""
+    return value
+
+
+with gr.Blocks() as demo:
+    gr.Markdown("## LaMaison — Floor Plan")
+    floor_plan = FloorPlan(value=DEFAULT_VALUE, label="Floor Plan", interactive=True)
+    output = gr.JSON(label="Updated positions")
+    floor_plan.change(on_furniture_moved, inputs=floor_plan, outputs=output)
+
+if __name__ == "__main__":
+    demo.launch()
+
+```
+
+## `FloorPlan`
+
+### Initialization
+
+<table>
+<thead>
+<tr>
+<th align="left">name</th>
+<th align="left" style="width: 25%;">type</th>
+<th align="left">default</th>
+<th align="left">description</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td align="left"><code>value</code></td>
+<td align="left" style="width: 25%;">
+
+```python
+dict | None
+```
+
+</td>
+<td align="left"><code>value = None</code></td>
+<td align="left">None</td>
+</tr>
+
+<tr>
+<td align="left"><code>label</code></td>
+<td align="left" style="width: 25%;">
+
+```python
+str | None
+```
+
+</td>
+<td align="left"><code>value = None</code></td>
+<td align="left">None</td>
+</tr>
+
+<tr>
+<td align="left"><code>info</code></td>
+<td align="left" style="width: 25%;">
+
+```python
+str | None
+```
+
+</td>
+<td align="left"><code>value = None</code></td>
+<td align="left">None</td>
+</tr>
+
+<tr>
+<td align="left"><code>every</code></td>
+<td align="left" style="width: 25%;">
+
+```python
+'Timer | float | None'
+```
+
+</td>
+<td align="left"><code>value = None</code></td>
+<td align="left">None</td>
+</tr>
+
+<tr>
+<td align="left"><code>show_label</code></td>
+<td align="left" style="width: 25%;">
+
+```python
+bool | None
+```
+
+</td>
+<td align="left"><code>value = None</code></td>
+<td align="left">None</td>
+</tr>
+
+<tr>
+<td align="left"><code>container</code></td>
+<td align="left" style="width: 25%;">
+
+```python
+bool
+```
+
+</td>
+<td align="left"><code>value = True</code></td>
+<td align="left">None</td>
+</tr>
+
+<tr>
+<td align="left"><code>scale</code></td>
+<td align="left" style="width: 25%;">
+
+```python
+int | None
+```
+
+</td>
+<td align="left"><code>value = None</code></td>
+<td align="left">None</td>
+</tr>
+
+<tr>
+<td align="left"><code>min_width</code></td>
+<td align="left" style="width: 25%;">
+
+```python
+int
+```
+
+</td>
+<td align="left"><code>value = 160</code></td>
+<td align="left">None</td>
+</tr>
+
+<tr>
+<td align="left"><code>interactive</code></td>
+<td align="left" style="width: 25%;">
+
+```python
+bool | None
 ```
+
+</td>
+<td align="left"><code>value = None</code></td>
+<td align="left">None</td>
+</tr>
+
+<tr>
+<td align="left"><code>visible</code></td>
+<td align="left" style="width: 25%;">
+
+```python
+bool
+```
+
+</td>
+<td align="left"><code>value = True</code></td>
+<td align="left">None</td>
+</tr>
+
+<tr>
+<td align="left"><code>elem_id</code></td>
+<td align="left" style="width: 25%;">
+
+```python
+str | None
+```
+
+</td>
+<td align="left"><code>value = None</code></td>
+<td align="left">None</td>
+</tr>
+
+<tr>
+<td align="left"><code>elem_classes</code></td>
+<td align="left" style="width: 25%;">
+
+```python
+list[str] | str | None
+```
+
+</td>
+<td align="left"><code>value = None</code></td>
+<td align="left">None</td>
+</tr>
+
+<tr>
+<td align="left"><code>render</code></td>
+<td align="left" style="width: 25%;">
+
+```python
+bool
+```
+
+</td>
+<td align="left"><code>value = True</code></td>
+<td align="left">None</td>
+</tr>
+</tbody></table>
+
+
+### Events
+
+| name | description |
+|:-----|:------------|
+| `change` |  |
+
+
+
+### User function
+
+The impact on the users predict function varies depending on whether the component is used as an input or output for an event (or both).
+
+- When used as an Input, the component only impacts the input signature of the user function.
+- When used as an output, the component only impacts the return signature of the user function.
+
+The code snippet below is accurate in cases where the component is used as both an input and an output.
+
+
+
+ ```python
+ def predict(
+     value: dict| None
+ ) -> dict| None:
+     return value
+ ```
+ 

floorplan/frontend/FloorPlan.svelte

@@ -44,6 +44,16 @@
   let startDy = 0;
   let savedOffsets: [number, number][] = [];
 
+  // View state
+  let zoom = 1;
+  let viewBoxPanX = 0;
+  let viewBoxPanY = 0;
+  let isPanning = false;
+  let panStartX = 0;
+  let panStartY = 0;
+  let panStartViewBoxX = 0;
+  let panStartViewBoxY = 0;
+
   function clampOffset(
     dx: number,
     dy: number,
@@ -58,15 +68,93 @@
   }
 
   function clientToSvg(clientX: number, clientY: number): [number, number] {
+    // Return coordinates mapped precisely taking zoom and pan into account
+    // (bounding box returns layout size in CSS pixels)
     const rect = svgEl.getBoundingClientRect();
+    
+    // Scale from CSS pixels to base SVG pixels
     const scaleX = SVG_WIDTH / rect.width;
     const scaleY = SVG_HEIGHT / rect.height;
-    return [
-      (clientX - rect.left) * scaleX,
-      (clientY - rect.top) * scaleY,
-    ];
+
+    // Apply viewport scale and translation (zoom & pan)
+    const baseSvgX = (clientX - rect.left) * scaleX;
+    const baseSvgY = (clientY - rect.top) * scaleY;
+
+    // Apply the inverse of the viewbox transform
+    const sx = viewBoxPanX + (baseSvgX / zoom);
+    const sy = viewBoxPanY + (baseSvgY / zoom);
+
+    return [sx, sy];
+  }
+
+  // --- Background Panning / Zooming ---
+
+  function onSvgPointerDown(e: PointerEvent) {
+    // If we clicked directly on the background (svgEl) or polygon
+    if (e.target === svgEl || (e.target as Element).tagName.toLowerCase() === 'polygon') {
+      isPanning = true;
+      panStartX = e.clientX;
+      panStartY = e.clientY;
+      panStartViewBoxX = viewBoxPanX;
+      panStartViewBoxY = viewBoxPanY;
+      svgEl.setPointerCapture(e.pointerId);
+    }
+  }
+
+  function onSvgPointerMove(e: PointerEvent) {
+    if (isPanning) {
+        const rect = svgEl.getBoundingClientRect();
+        const scaleX = SVG_WIDTH / rect.width;
+        const scaleY = SVG_HEIGHT / rect.height;
+
+        const dxStr = (e.clientX - panStartX) * scaleX;
+        const dyStr = (e.clientY - panStartY) * scaleY;
+
+        viewBoxPanX = panStartViewBoxX - (dxStr / zoom);
+        viewBoxPanY = panStartViewBoxY - (dyStr / zoom);
+    } else {
+        onPointerMove(e);
+    }
+  }
+
+  function onSvgPointerUp(e: PointerEvent) {
+      if (isPanning) {
+          isPanning = false;
+          svgEl.releasePointerCapture(e.pointerId);
+      } else {
+          if (e.type === 'pointercancel') {
+             onPointerCancel(e);
+          } else {
+             onPointerUp(e);
+          }
+      }
   }
 
+  function onWheel(e: WheelEvent) {
+      // Zoom centered at pointer
+      const zoomFactor = 1.1;
+      const direction = e.deltaY > 0 ? -1 : 1;
+      
+      const [pointerSvgX, pointerSvgY] = clientToSvg(e.clientX, e.clientY);
+
+      if (direction === 1) {
+          zoom *= zoomFactor;
+      } else {
+          zoom /= zoomFactor;
+      }
+
+      // Constrain zoom to reasonable bounds
+      zoom = Math.max(0.2, Math.min(zoom, 10));
+
+      // Re-center around pointer position
+      const [newPointerSvgX, newPointerSvgY] = clientToSvg(e.clientX, e.clientY);
+      viewBoxPanX -= (newPointerSvgX - pointerSvgX);
+      viewBoxPanY -= (newPointerSvgY - pointerSvgY);
+  }
+
+  // --- Object Dragging ---
+
+
   function onPointerDown(e: PointerEvent, i: number) {
     if (!interactive) return;
     e.preventDefault();
@@ -125,12 +213,15 @@
 <!-- svelte-ignore a11y-no-static-element-interactions -->
 <svg
   bind:this={svgEl}
-  width={SVG_WIDTH}
-  height={SVG_HEIGHT}
-  style="border: 1px solid #ccc; background: #fafafa; display: block;"
-  on:pointermove={onPointerMove}
-  on:pointerup={onPointerUp}
-  on:pointercancel={onPointerCancel}
+  width="100%"
+  height="100%"
+  viewBox="{viewBoxPanX} {viewBoxPanY} {SVG_WIDTH / zoom} {SVG_HEIGHT / zoom}"
+  style="border: 1px solid #ccc; background: #fafafa; display: block; min-height: 400px; cursor: {isPanning ? 'grabbing' : 'grab'}; touch-action: none;"
+  on:pointerdown={onSvgPointerDown}
+  on:pointermove={onSvgPointerMove}
+  on:pointerup={onSvgPointerUp}
+  on:pointercancel={onSvgPointerUp}
+  on:wheel|preventDefault={onWheel}
 >
   <!-- Room outline -->
   <polygon points={polygonPoints} fill="none" stroke="#333" stroke-width="2" />

floorplan/frontend/package-lock.json

@@ -16,7 +16,7 @@
         "svelte": "^5.48.0"
       },
       "devDependencies": {
-        "@gradio/preview": "0.16.0",
+        "@gradio/preview": "^0.16.0",
         "vitest": "^4.1.0"
       },
       "peerDependencies": {

floorplan/frontend/package.json

@@ -28,7 +28,7 @@
     "svelte": "^5.48.0"
   },
   "devDependencies": {
-    "@gradio/preview": "0.16.0",
+    "@gradio/preview": "^0.16.0",
     "vitest": "^4.1.0"
   },
   "peerDependencies": {

floorplan/pyproject.toml

@@ -45,7 +45,7 @@ classifiers = [
 dev = ["build", "twine"]
 
 [tool.hatch.build]
-artifacts = ["/backend/gradio_floorplan/templates", "*.pyi"]
+artifacts = ["/backend/gradio_floorplan/templates", "*.pyi", "/opt/homebrew/lib/python3.14/site-packages/gradio_floorplan/templates"]
 
 [tool.hatch.build.targets.wheel]
 packages = ["/backend/gradio_floorplan"]

requirements.txt

@@ -0,0 +1,4 @@
+google-genai
+python-dotenv
+pillow
+pydantic