Upload 2 files

Fixed issue with Q&A

README.md

@@ -8,7 +8,7 @@ pinned: false
 license: mit
 models:
   - Xenova/vit-gpt2-image-captioning
-  - Xenova/blip-vqa-base
+  - Xenova/owlvit-base-patch32
   - Xenova/clip-vit-base-patch32
   - Xenova/trocr-small-printed
 ---
@@ -28,7 +28,7 @@ acceleration when the browser supports it.
 | Tab | Task | Model |
 |---|---|---|
 | Caption | Image → natural-language description | `Xenova/vit-gpt2-image-captioning` |
-| Visual Q&A | Image + free-form question → answer | `Xenova/blip-vqa-base` |
+| Detect Objects | Draws bounding boxes for any labels you define, even ones the model wasn't explicitly trained to detect | `Xenova/owlvit-base-patch32` |
 | Classify | Zero-shot classification against any labels you define | `Xenova/clip-vit-base-patch32` |
 | Read Text | OCR on printed text in an image | `Xenova/trocr-small-printed` |
 | Semantic Search | Rank a gallery of images against a natural-language query | `Xenova/clip-vit-base-patch32` (shared) |
@@ -66,6 +66,16 @@ python -m http.server 8000
 ## Extending it
 
 Ideas for going further: add a webcam capture option alongside file upload,
-swap in a larger VQA or captioning model for better quality (trading off
-download size), or add an "export gallery as JSON" button to persist search
-results across sessions.
+swap in a larger captioning model for better quality (trading off download
+size), or add an "export gallery as JSON" button to persist search results
+across sessions.
+
+## A note on transformers.js task coverage
+
+Not every task in the Python `transformers` library has a browser
+equivalent. Visual question answering, for instance, isn't currently a
+supported transformers.js pipeline — only its listed NLP, vision, audio,
+and multimodal tasks (embeddings, zero-shot classification/detection) run
+client-side. Object detection was used here instead of VQA for exactly that
+reason: it's a confirmed, supported multimodal task that still combines
+free-form text labels with an image.

index.html

@@ -301,6 +301,32 @@
     100% { top: 4%; }
   }
 
+  .detect-boxes {
+    position: absolute;
+    inset: 0;
+    pointer-events: none;
+  }
+  .detect-box {
+    position: absolute;
+    border: 2px solid var(--coral);
+    box-shadow: 0 0 0 1px rgba(0,0,0,0.5);
+    border-radius: 2px;
+  }
+  .detect-box .tag {
+    position: absolute;
+    top: -20px;
+    left: -2px;
+    background: var(--coral);
+    color: #1a0a06;
+    font-family: var(--mono);
+    font-size: 10px;
+    font-weight: 600;
+    padding: 1px 6px;
+    border-radius: 3px;
+    white-space: nowrap;
+  }
+  .detect-box.tag-inside .tag { top: 2px; left: 2px; }
+
   .vf-replace {
     position: absolute;
     bottom: 10px;
@@ -599,7 +625,7 @@
 <div class="layout">
   <nav class="tabs" id="tabNav">
     <button class="tab-btn active" data-tab="caption"><span class="idx">01</span> Caption</button>
-    <button class="tab-btn" data-tab="vqa"><span class="idx">02</span> Visual Q&amp;A</button>
+    <button class="tab-btn" data-tab="detect"><span class="idx">02</span> Detect Objects</button>
     <button class="tab-btn" data-tab="classify"><span class="idx">03</span> Classify</button>
     <button class="tab-btn" data-tab="ocr"><span class="idx">04</span> Read Text</button>
     <button class="tab-btn" data-tab="search"><span class="idx">05</span> Semantic Search</button>
@@ -622,6 +648,7 @@
           <div class="bracket bl"></div>
           <div class="bracket br"></div>
           <div class="scanline" id="scanline"></div>
+          <div class="detect-boxes" id="detectBoxes"></div>
           <div class="vf-replace">click to replace</div>
         </div>
       </div>
@@ -645,21 +672,21 @@
       </div>
     </section>
 
-    <!-- Panel: VQA -->
-    <section class="panel" data-panel="vqa">
+    <!-- Panel: Detect -->
+    <section class="panel" data-panel="detect">
       <div class="panel-head">
-        <h2>Visual Question Answering</h2>
-        <p>Ask a free-form question about the image; the model fuses the visual and text encoders to produce an answer.</p>
-        <span class="model-tag">model: Xenova/blip-vqa-base</span>
+        <h2>Zero-Shot Object Detection</h2>
+        <p>Define any set of objects to look for — the model locates and draws bounding boxes around them in the image above, even for categories it wasn't explicitly trained to detect.</p>
+        <span class="model-tag">model: Xenova/owlvit-base-patch32</span>
       </div>
       <div class="controls">
-        <input type="text" id="vqaQuestion" placeholder="What color is the object in the center?" />
-        <button class="run" id="btnVqa" disabled>Ask</button>
+        <input type="text" id="detectLabels" placeholder="person, dog, car, tree, building" value="person, dog, car, tree, building" />
+        <button class="run" id="btnDetect" disabled>Detect objects</button>
       </div>
       <div class="output">
-        <div class="output-card" id="outVqa">
-          <div class="output-label">Answer</div>
-          <div class="output-text" id="vqaText"></div>
+        <div class="output-card" id="outDetect">
+          <div class="output-label">Detections</div>
+          <div id="detectList"></div>
         </div>
       </div>
     </section>
@@ -795,7 +822,7 @@
   /* ---------------- Model registry ---------------- */
   const MODEL_SPECS = {
     caption: { task: "image-to-text", model: "Xenova/vit-gpt2-image-captioning" },
-    vqa: { task: "visual-question-answering", model: "Xenova/blip-vqa-base" },
+    detect: { task: "zero-shot-object-detection", model: "Xenova/owlvit-base-patch32" },
     classify: { task: "zero-shot-image-classification", model: "Xenova/clip-vit-base-patch32" },
     ocr: { task: "image-to-text", model: "Xenova/trocr-small-printed" },
   };
@@ -865,7 +892,7 @@
   let currentImageURL = null;
   const actionButtons = [
     document.getElementById("btnCaption"),
-    document.getElementById("btnVqa"),
+    document.getElementById("btnDetect"),
     document.getElementById("btnClassify"),
     document.getElementById("btnOcr"),
   ];
@@ -876,6 +903,7 @@
     vfEmpty.style.display = "none";
     vfImageWrap.style.display = "block";
     actionButtons.forEach((b) => (b.disabled = false));
+    clearDetectionBoxes();
     log("Image loaded into viewfinder");
   }
 
@@ -943,31 +971,82 @@
     }
   });
 
-  /* ---------------- VQA ---------------- */
-  const btnVqa = document.getElementById("btnVqa");
-  const vqaQuestion = document.getElementById("vqaQuestion");
-  const outVqa = document.getElementById("outVqa");
-  const vqaText = document.getElementById("vqaText");
+  /* ---------------- Detect (zero-shot object detection) ---------------- */
+  const btnDetect = document.getElementById("btnDetect");
+  const detectLabels = document.getElementById("detectLabels");
+  const outDetect = document.getElementById("outDetect");
+  const detectList = document.getElementById("detectList");
+  const detectBoxesEl = document.getElementById("detectBoxes");
+  let lastDetections = [];
+
+  function clearDetectionBoxes() {
+    lastDetections = [];
+    detectBoxesEl.innerHTML = "";
+  }
+
+  function renderDetectionBoxes() {
+    detectBoxesEl.innerHTML = "";
+    if (!lastDetections.length || !vfImage.naturalWidth) return;
+    const containerW = vfImageWrap.clientWidth;
+    const containerH = vfImageWrap.clientHeight;
+    const naturalW = vfImage.naturalWidth;
+    const naturalH = vfImage.naturalHeight;
+    const scale = Math.min(containerW / naturalW, containerH / naturalH);
+    const renderW = naturalW * scale;
+    const renderH = naturalH * scale;
+    const offsetX = (containerW - renderW) / 2;
+    const offsetY = (containerH - renderH) / 2;
+
+    lastDetections.forEach((d) => {
+      const box = document.createElement("div");
+      const top = offsetY + d.box.ymin * scale;
+      box.className = "detect-box" + (top < 22 ? " tag-inside" : "");
+      box.style.left = (offsetX + d.box.xmin * scale) + "px";
+      box.style.top = top + "px";
+      box.style.width = ((d.box.xmax - d.box.xmin) * scale) + "px";
+      box.style.height = ((d.box.ymax - d.box.ymin) * scale) + "px";
+      const tag = document.createElement("div");
+      tag.className = "tag";
+      tag.textContent = `${d.label} ${(d.score * 100).toFixed(0)}%`;
+      box.appendChild(tag);
+      detectBoxesEl.appendChild(box);
+    });
+  }
 
-  btnVqa.addEventListener("click", async () => {
+  let resizeTimer;
+  window.addEventListener("resize", () => {
+    clearTimeout(resizeTimer);
+    resizeTimer = setTimeout(renderDetectionBoxes, 120);
+  });
+
+  btnDetect.addEventListener("click", async () => {
     if (!currentImageURL) return;
-    const question = vqaQuestion.value.trim() || "What is in this image?";
-    btnVqa.disabled = true;
+    const labels = detectLabels.value.split(",").map((s) => s.trim()).filter(Boolean);
+    if (!labels.length) {
+      log("Enter at least one label to detect", "warn");
+      return;
+    }
+    btnDetect.disabled = true;
     withScan(true);
-    setBusy(true, "answering…");
+    setBusy(true, "detecting objects…");
     try {
-      const vqa = await getPipeline("vqa");
-      const result = await vqa(currentImageURL, question);
-      const answer = Array.isArray(result) ? result[0]?.answer : result?.answer;
-      vqaText.textContent = answer || "(no answer produced)";
-      outVqa.classList.add("show");
-      log(`VQA answered: "${question}"`, "ok");
+      const detector = await getPipeline("detect");
+      const output = await detector(currentImageURL, labels, { threshold: 0.12, top_k: 12 });
+      lastDetections = output;
+      renderDetectionBoxes();
+
+      detectList.innerHTML = "";
+      if (!output.length) {
+        detectList.innerHTML = `<div class="output-text" style="font-size:14px; color: var(--ink-dim);">No objects matched those labels above the confidence threshold.</div>`;
+      } else {
+        renderBars(detectList, [...output].sort((a, b) => b.score - a.score));
+      }
+      outDetect.classList.add("show");
+      log(`Detected ${output.length} object(s) across ${labels.length} label(s)`, "ok");
     } catch (err) {
-      vqaText.textContent = "Error answering question — see console.";
-      outVqa.classList.add("show");
-      log(`VQA error: ${err.message || err}`, "err");
+      log(`Detect error: ${err.message || err}`, "err");
     } finally {
-      btnVqa.disabled = false;
+      btnDetect.disabled = false;
       withScan(false);
       setBusy(false);
     }