feat(ui): manual garment crop with Annotorious bounding-box editor

Upload a photo → server auto-detects garments via YOLOS and returns
bounding boxes → Annotorious v3.8.6 renders the image with the boxes
pre-drawn → user adds/edits/deletes boxes → "Analyse N garments"
crops each box server-side and runs the VLM on each crop.

Backend (app.py):
- Mount data/_uploads/ as /uploads (temp image store).
- api_prepare_image: saves upload, runs detect_garments, returns
token + image_url + detected boxes [{x,y,w,h}].
- api_analyze_boxes: validates token, crops each box via PIL,
calls extract_from_crop_bytes per crop, adds to catalog, cleans up.

Frontend (index.html):
- Load @annotorious/annotorious@3.8.6 CSS + ESM (version pinned).
- handleFile now calls /prepare_image then opens the editor.
- initAnnotator: creates annotator, loads initial boxes as W3C
FragmentSelector annotations, syncs box count on create/delete.
- analyzeBoxes: reads annotations, maps to pixel coords, calls
/analyze_boxes, refreshes wardrobe on success.
- cancelEditor: destroys annotator, resets editor state.

Co-authored-by: Cursor <cursoragent@cursor.com>

app.py

@@ -813,6 +813,10 @@ def _build_custom_server():
 
     server.mount("/garments", StaticFiles(directory=str(_DATA_DIR / "garments")), name="garments")
 
+    _UPLOADS_DIR = _DATA_DIR / "_uploads"
+    _UPLOADS_DIR.mkdir(parents=True, exist_ok=True)
+    server.mount("/uploads", StaticFiles(directory=str(_UPLOADS_DIR)), name="uploads")
+
     def _image_url(garment_id: str) -> str:
         """Return a cache-busted URL for a garment image using file mtime."""
         img_path = get_image_path(garment_id)
@@ -825,6 +829,113 @@ def _build_custom_server():
             v = 0
         return f"/garments/{garment_id}.jpg?v={v}"
 
+    @server.api(name="prepare_image")
+    def api_prepare_image(image_path: str | dict) -> dict:
+        """Save image to _uploads, run auto-detection, return token + detected boxes.
+
+        The frontend uses the token + image_url to show the image in Annotorious
+        pre-populated with auto-detected boxes for the user to review/edit.
+        """
+        import uuid
+        import time as _time
+
+        if isinstance(image_path, dict):
+            image_path = image_path.get("path") or image_path.get("url", "")
+
+        try:
+            img = Image.open(str(image_path)).convert("RGB")
+        except Exception as e:
+            return {"error": str(e), "token": "", "image_url": "", "width": 0, "height": 0, "boxes": []}
+
+        img.thumbnail((1280, 1280), Image.LANCZOS)
+        w, h = img.size
+
+        token = uuid.uuid4().hex
+        upload_path = _UPLOADS_DIR / f"{token}.jpg"
+        img.save(str(upload_path), format="JPEG", quality=90)
+
+        try:
+            boxes = detect_boxes(str(upload_path))
+        except Exception:
+            boxes = []
+
+        ts = int(_time.time())
+        return {
+            "token": token,
+            "image_url": f"/uploads/{token}.jpg?v={ts}",
+            "width": w,
+            "height": h,
+            "boxes": [{"x": b.x1, "y": b.y1, "w": b.width, "h": b.height} for b in boxes],
+        }
+
+    @server.api(name="analyze_boxes")
+    def api_analyze_boxes(token: str, boxes: str) -> dict:
+        """Crop each user-confirmed bounding box from the uploaded image and extract garment attributes.
+
+        Args:
+            token: filename token returned by prepare_image (hex, no path separators).
+            boxes: JSON string — list of {x, y, w, h} in pixels of the stored image.
+        """
+        import json as _json
+        import re as _re
+
+        # Validate token: only hex characters, no path traversal
+        if not _re.fullmatch(r"[0-9a-f]{32}", token):
+            return {"error": "Invalid token", "count": 0, "garments": []}
+
+        upload_path = _UPLOADS_DIR / f"{token}.jpg"
+        if not upload_path.exists():
+            return {"error": "Image not found. Please re-upload.", "count": 0, "garments": []}
+
+        try:
+            box_list = _json.loads(boxes)
+        except Exception:
+            return {"error": "Invalid boxes format", "count": 0, "garments": []}
+
+        try:
+            img = Image.open(str(upload_path)).convert("RGB")
+        except Exception as e:
+            return {"error": str(e), "count": 0, "garments": []}
+
+        results: list[tuple[dict, bytes]] = []
+        for box in box_list:
+            try:
+                x = int(box["x"])
+                y = int(box["y"])
+                w = int(box["w"])
+                h = int(box["h"])
+            except (KeyError, TypeError, ValueError):
+                continue
+
+            if w < 10 or h < 10:
+                continue
+
+            cropped = img.crop((x, y, x + w, y + h))
+            cropped.thumbnail((512, 512), Image.LANCZOS)
+
+            buf = io.BytesIO()
+            cropped.save(buf, format="JPEG", quality=85)
+            crop_bytes = buf.getvalue()
+
+            result = extract_from_crop_bytes(crop_bytes)
+            if result:
+                results.append(result)
+
+        # Cleanup upload temp file
+        try:
+            upload_path.unlink()
+        except OSError:
+            pass
+
+        if not results:
+            return {"error": "No garments could be extracted from the selections.", "count": 0, "garments": []}
+
+        added = add_garments(results)
+        for g in added:
+            g["image_url"] = _image_url(g["id"])
+
+        return {"count": len(added), "garments": added}
+
     @server.api(name="get_wardrobe")
     def api_get_wardrobe() -> dict:
         catalog = load_catalog()

src/ui/index.html

@@ -5,11 +5,14 @@
   <meta name="viewport" content="width=device-width, initial-scale=1.0">
   <title>Wardrobe AI</title>
   <link rel="stylesheet" href="/style.css">
+  <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@annotorious/annotorious@3.8.6/dist/annotorious.css">
   <script src="https://cdn.jsdelivr.net/npm/marked/marked.min.js"></script>
   <script defer src="https://cdn.jsdelivr.net/npm/alpinejs@3/dist/cdn.min.js"></script>
   <script type="module">
     import { Client, handle_file } from "https://cdn.jsdelivr.net/npm/@gradio/client/dist/index.min.js";
+    import { createImageAnnotator } from "https://cdn.jsdelivr.net/npm/@annotorious/annotorious@3.8.6/+esm";
     window.gradioClient = null;
+    window.annotoriousFactory = createImageAnnotator;
     Client.connect(window.location.origin).then(c => { window.gradioClient = c; });
   </script>
 </head>
@@ -107,6 +110,24 @@
 
       <div class="status" :class="uploadStatus.type" x-show="uploadStatus.message" x-text="uploadStatus.message"></div>
 
+      <!-- Bounding box editor (shown after upload) -->
+      <div class="editor-area" x-show="editorActive">
+        <p class="editor-hint">
+          <strong x-text="editorAutoBoxCount > 0 ? `${editorAutoBoxCount} garment${editorAutoBoxCount > 1 ? 's' : ''} auto-detected.` : 'No garments auto-detected.'"></strong>
+          Draw rectangles over any missing ones, adjust or delete existing boxes, then click Analyse.
+        </p>
+        <div class="editor-canvas-wrap">
+          <img id="annoImage" :src="editorImageUrl" alt="Garment photo">
+        </div>
+        <div class="editor-actions">
+          <button class="btn btn-primary" @click="analyzeBoxes()" :disabled="analyzing || editorBoxCount === 0">
+            <span x-text="analyzing ? 'Analysing…' : `Analyse ${editorBoxCount} garment${editorBoxCount !== 1 ? 's' : ''}`"></span>
+          </button>
+          <button class="btn btn-secondary" @click="cancelEditor()" :disabled="analyzing">Cancel</button>
+        </div>
+        <div class="status" :class="analyzeStatus.type" x-show="analyzeStatus.message" x-text="analyzeStatus.message"></div>
+      </div>
+
       <div style="margin-top:2rem">
         <h2>Or load a sample dataset</h2>
         <div class="dataset-row">
@@ -220,6 +241,16 @@
         // Garment detail
         selectedGarment: null,
 
+        // Bounding-box editor
+        editorActive: false,
+        editorImageUrl: '',
+        editorToken: '',
+        editorBoxCount: 0,
+        editorAutoBoxCount: 0,
+        analyzing: false,
+        analyzeStatus: { type: '', message: '' },
+        anno: null,
+
         // Outfits
         context: '',
         outfits: [],
@@ -252,17 +283,28 @@
 
         async handleFile(file) {
           if (!file) return;
-          this.uploadStatus = { type: 'loading', message: 'Analyzing your clothes...' };
+          this.uploadStatus = { type: 'loading', message: 'Detecting garments…' };
+          this.editorActive = false;
+          this.analyzeStatus = { type: '', message: '' };
+
           try {
             const { handle_file } = await import("https://cdn.jsdelivr.net/npm/@gradio/client/dist/index.min.js");
-            const result = await window.gradioClient.predict("/add_photo", { image_path: handle_file(file) });
+            const result = await window.gradioClient.predict("/prepare_image", { image_path: handle_file(file) });
             const data = result.data[0];
-            if (data.count > 0) {
-              this.uploadStatus = { type: 'success', message: `Found ${data.count} garment${data.count > 1 ? 's' : ''}! Check your wardrobe.` };
-              await this.loadWardrobe();
-            } else {
-              this.uploadStatus = { type: 'error', message: "Couldn't detect any garments. Try a clearer image." };
+
+            if (data.error) {
+              this.uploadStatus = { type: 'error', message: data.error };
+              return;
             }
+
+            this.uploadStatus = { type: '', message: '' };
+            this.editorToken = data.token;
+            this.editorImageUrl = data.image_url;
+            this.editorAutoBoxCount = data.boxes.length;
+            this.editorBoxCount = data.boxes.length;
+            this.editorActive = true;
+
+            this.$nextTick(() => this.initAnnotator(data.boxes, data.width, data.height));
           } catch (e) {
             this.uploadStatus = { type: 'error', message: 'Something went wrong. Please try again.' };
             console.error(e);
@@ -385,6 +427,109 @@
           return div.innerHTML;
         },
 
+        initAnnotator(boxes, imgW, imgH) {
+          // Destroy any previous instance
+          if (this.anno) {
+            try { this.anno.destroy(); } catch (_) {}
+            this.anno = null;
+          }
+
+          const factory = window.annotoriousFactory;
+          if (!factory) {
+            console.error('Annotorious not loaded yet');
+            return;
+          }
+
+          const el = document.getElementById('annoImage');
+          if (!el) return;
+
+          this.anno = factory(el, {
+            drawingEnabled: true,
+            drawingTool: 'rectangle',
+            style: { fill: '#3b82f6', fillOpacity: 0.15, stroke: '#3b82f6', strokeWidth: 2 },
+          });
+
+          // Load auto-detected boxes as initial annotations (W3C Web Annotation format)
+          if (boxes.length > 0) {
+            const annotations = boxes.map((b, i) => ({
+              id: `auto-${i}`,
+              type: 'Annotation',
+              target: {
+                selector: {
+                  type: 'FragmentSelector',
+                  conformsTo: 'http://www.w3.org/TR/media-frags/',
+                  value: `xywh=pixel:${b.x},${b.y},${b.w},${b.h}`,
+                },
+              },
+              body: [],
+            }));
+            this.anno.setAnnotations(annotations);
+          }
+
+          // Keep editorBoxCount in sync
+          const updateCount = () => {
+            this.editorBoxCount = this.anno.getAnnotations().length;
+          };
+          this.anno.on('createAnnotation', updateCount);
+          this.anno.on('deleteAnnotation', updateCount);
+          this.anno.on('updateAnnotation', updateCount);
+        },
+
+        async analyzeBoxes() {
+          if (!this.anno || this.editorBoxCount === 0) return;
+          this.analyzing = true;
+          this.analyzeStatus = { type: 'loading', message: 'Extracting garment attributes…' };
+
+          // Map W3C annotations to {x,y,w,h} pixel boxes
+          const annotations = this.anno.getAnnotations();
+          const boxes = annotations.map(ann => {
+            // FragmentSelector: "xywh=pixel:x,y,w,h"
+            const sel = ann.target?.selector;
+            if (sel?.type === 'FragmentSelector') {
+              const m = sel.value.match(/xywh=pixel:([\d.]+),([\d.]+),([\d.]+),([\d.]+)/);
+              if (m) return { x: Math.round(+m[1]), y: Math.round(+m[2]), w: Math.round(+m[3]), h: Math.round(+m[4]) };
+            }
+            return null;
+          }).filter(Boolean);
+
+          if (boxes.length === 0) {
+            this.analyzeStatus = { type: 'error', message: 'No valid boxes to analyze.' };
+            this.analyzing = false;
+            return;
+          }
+
+          try {
+            const result = await window.gradioClient.predict("/analyze_boxes", {
+              token: this.editorToken,
+              boxes: JSON.stringify(boxes),
+            });
+            const data = result.data[0];
+            if (data.error) {
+              this.analyzeStatus = { type: 'error', message: data.error };
+            } else {
+              this.analyzeStatus = { type: 'success', message: `Added ${data.count} garment${data.count !== 1 ? 's' : ''} to your wardrobe!` };
+              await this.loadWardrobe();
+              this.cancelEditor();
+            }
+          } catch (e) {
+            this.analyzeStatus = { type: 'error', message: 'Something went wrong. Please try again.' };
+            console.error(e);
+          }
+          this.analyzing = false;
+        },
+
+        cancelEditor() {
+          if (this.anno) {
+            try { this.anno.destroy(); } catch (_) {}
+            this.anno = null;
+          }
+          this.editorActive = false;
+          this.editorToken = '';
+          this.editorImageUrl = '';
+          this.editorBoxCount = 0;
+          this.editorAutoBoxCount = 0;
+        },
+
         selectGarment(g) {
           this.selectedGarment = g;
         },

src/ui/style.css

@@ -629,6 +629,44 @@ nav button.active {
   font-weight: 500;
 }
 
+/* Bounding box editor */
+.editor-area {
+  margin-top: 1.5rem;
+}
+
+.editor-hint {
+  font-size: 0.85rem;
+  color: var(--text-muted);
+  margin-bottom: 0.85rem;
+  line-height: 1.5;
+}
+
+.editor-hint strong {
+  color: var(--text);
+}
+
+.editor-canvas-wrap {
+  position: relative;
+  max-height: 60vh;
+  overflow: auto;
+  border: 1px solid var(--border);
+  border-radius: var(--radius-sm);
+  background: #f3f4f6;
+}
+
+.editor-canvas-wrap #annoImage {
+  display: block;
+  max-width: 100%;
+  height: auto;
+}
+
+.editor-actions {
+  display: flex;
+  gap: 0.75rem;
+  margin-top: 1rem;
+  flex-wrap: wrap;
+}
+
 /* Log dock — fixed bottom panel */
 .log-dock {
   position: fixed;