Update app.py

app.py

@@ -7,7 +7,6 @@ import tempfile
 import asyncio
 import threading
 import time
-import numpy as np
 import base64
 from io import BytesIO
 
@@ -56,9 +55,20 @@ def text_to_speech(text: str) -> str:
     return path
 
 
-def caption_from_pil(image: Image.Image, task_choice: str):
+def describe_frame(frame_b64: str, task_choice: str):
+    """Called every 3s by JS via hidden button. Receives raw base64 jpeg."""
+    if not frame_b64 or frame_b64 == "none" or "," not in frame_b64:
+        return gr.update(), gr.update()
+    try:
+        img_bytes = base64.b64decode(frame_b64.split(",")[1])
+        image = Image.open(BytesIO(img_bytes)).convert("RGB")
+    except Exception as e:
+        print(f"Decode error: {e}")
+        return gr.update(), gr.update()
+
     h = image_hash(image)
     if h == last_caption["hash"] and last_caption["text"]:
+        print("Same frame, skipping.")
         return gr.update(), gr.update()
 
     task_map = {
@@ -69,7 +79,6 @@ def caption_from_pil(image: Image.Image, task_choice: str):
 
     t0 = time.time()
     inputs = processor(text=task, images=image, return_tensors="pt").to(device)
-
     with torch.inference_mode():
         output_ids = model.generate(
             input_ids=inputs["input_ids"],
@@ -78,15 +87,13 @@ def caption_from_pil(image: Image.Image, task_choice: str):
             do_sample=False,
             num_beams=1,
         )
-
     generated_text = processor.batch_decode(output_ids, skip_special_tokens=False)[0]
     result = processor.post_process_generation(
-        generated_text,
-        task=task,
+        generated_text, task=task,
         image_size=(image.width, image.height),
     )
     caption = result[task]
-    print(f"Caption ({time.time()-t0:.2f}s): {caption}")
+    print(f"[{time.time()-t0:.2f}s] {caption}")
 
     last_caption["text"] = caption
     last_caption["hash"] = h
@@ -95,37 +102,14 @@ def caption_from_pil(image: Image.Image, task_choice: str):
     return caption, audio_path
 
 
-def describe_frame(frame_b64: str, task_choice: str):
-    """Receives base64 jpeg from JS webcam snapshot."""
-    if not frame_b64 or frame_b64 == "none":
-        return gr.update(), gr.update()
-    try:
-        # Strip data URL header if present
-        if "," in frame_b64:
-            frame_b64 = frame_b64.split(",")[1]
-        img_bytes = base64.b64decode(frame_b64)
-        image = Image.open(BytesIO(img_bytes)).convert("RGB")
-        return caption_from_pil(image, task_choice)
-    except Exception as e:
-        print(f"Frame error: {e}")
-        return gr.update(), gr.update()
-
-
-def describe_upload(image, task_choice):
-    """Streaming version for manual/upload."""
+def describe_once(image, task_choice):
+    """Manual describe with word streaming."""
     if image is None:
-        yield "Please upload or capture an image.", None
+        yield "Please open the camera first.", None
         return
-
     if not isinstance(image, Image.Image):
         image = Image.fromarray(image)
 
-    h = image_hash(image)
-    if h == last_caption["hash"] and last_caption["text"]:
-        audio_path = text_to_speech(last_caption["text"])
-        yield last_caption["text"], audio_path
-        return
-
     task_map = {
         "Quick (faster)": "<CAPTION>",
         "Detailed (slower)": "<MORE_DETAILED_CAPTION>",
@@ -133,7 +117,6 @@ def describe_upload(image, task_choice):
     task = task_map.get(task_choice, "<CAPTION>")
 
     inputs = processor(text=task, images=image, return_tensors="pt").to(device)
-
     with torch.inference_mode():
         output_ids = model.generate(
             input_ids=inputs["input_ids"],
@@ -142,92 +125,127 @@ def describe_upload(image, task_choice):
             do_sample=False,
             num_beams=1,
         )
-
     generated_text = processor.batch_decode(output_ids, skip_special_tokens=False)[0]
     result = processor.post_process_generation(
-        generated_text,
-        task=task,
+        generated_text, task=task,
         image_size=(image.width, image.height),
     )
     caption = result[task]
-
     last_caption["text"] = caption
-    last_caption["hash"] = h
+    last_caption["hash"] = image_hash(image)
 
-    words = caption.split()
-    partial = ""
-    for word in words:
-        partial += ("" if partial == "" else " ") + word
-        yield partial, None
+    for word in caption.split():
+        yield (caption[:caption.index(word) + len(word)]), None
 
     audio_path = text_to_speech(caption)
     yield caption, audio_path
 
 
-# JS captures webcam frame → puts base64 in hidden textbox → triggers hidden button
+def describe_upload(image, task_choice):
+    """Upload image describe with streaming."""
+    if image is None:
+        yield "Please upload an image.", None
+        return
+    yield from describe_once(image, task_choice)
+
+
 WEBCAM_JS = """
 <script>
-let realtimeTimer = null;
+(function() {
+    let realtimeTimer = null;
+    let isRunning = false;
+
+    function getVideo() {
+        // Gradio renders webcam inside shadow DOM or iframe — try all videos
+        const videos = document.querySelectorAll('video');
+        for (const v of videos) {
+            if (v.videoWidth > 0 && v.readyState >= 2) return v;
+        }
+        return null;
+    }
 
-function captureAndSend() {
-    const video = document.querySelector('video');
-    if (!video || video.readyState < 2) {
-        console.log('Video not ready');
-        return;
+    function setNativeValue(el, value) {
+        const setter = Object.getOwnPropertyDescriptor(
+            window.HTMLTextAreaElement.prototype, 'value'
+        ).set;
+        setter.call(el, value);
+        el.dispatchEvent(new Event('input', { bubbles: true }));
     }
-    const canvas = document.createElement('canvas');
-    canvas.width = video.videoWidth || 640;
-    canvas.height = video.videoHeight || 480;
-    canvas.getContext('2d').drawImage(video, 0, 0);
-    const b64 = canvas.toDataURL('image/jpeg', 0.7);
-
-    // Put base64 into hidden textbox
-    const hiddenBox = document.querySelector('#frame-input textarea');
-    if (!hiddenBox) { console.log('No hidden box'); return; }
-
-    // Set value via React/Svelte-compatible input event
-    const nativeInputValueSetter = Object.getOwnPropertyDescriptor(window.HTMLTextAreaElement.prototype, 'value').set;
-    nativeInputValueSetter.call(hiddenBox, b64);
-    hiddenBox.dispatchEvent(new Event('input', { bubbles: true }));
-
-    // Click hidden submit button
-    setTimeout(() => {
-        const btn = document.querySelector('#frame-submit');
-        if (btn) btn.click();
-        else console.log('No submit btn');
-    }, 100);
-}
-
-function startRealtime() {
-    if (realtimeTimer) return;
-    console.log('Realtime started');
-    captureAndSend(); // immediate first capture
-    realtimeTimer = setInterval(captureAndSend, 3000);
-}
-
-function stopRealtime() {
-    if (realtimeTimer) {
-        clearInterval(realtimeTimer);
-        realtimeTimer = null;
-        console.log('Realtime stopped');
+
+    function captureAndSend() {
+        const video = getVideo();
+        if (!video) {
+            console.warn('[EchoLens] No active video found');
+            return;
+        }
+        const canvas = document.createElement('canvas');
+        canvas.width  = video.videoWidth;
+        canvas.height = video.videoHeight;
+        canvas.getContext('2d').drawImage(video, 0, 0);
+        const b64 = canvas.toDataURL('image/jpeg', 0.75);
+
+        const box = document.querySelector('#frame-box textarea');
+        if (!box) { console.warn('[EchoLens] No frame-box textarea'); return; }
+        setNativeValue(box, b64);
+
+        setTimeout(() => {
+            const btn = document.querySelector('#frame-btn button');
+            if (btn) {
+                btn.click();
+                console.log('[EchoLens] Frame sent');
+            } else {
+                console.warn('[EchoLens] No frame-btn button');
+            }
+        }, 150);
     }
-}
 
-// Expose globally so Gradio buttons can call them
-window.startRealtime = startRealtime;
-window.stopRealtime = stopRealtime;
+    window.echoToggleRealtime = function() {
+        const toggleBtn = document.querySelector('#rt-btn button');
+        if (!isRunning) {
+            isRunning = true;
+            if (toggleBtn) {
+                toggleBtn.textContent = '⏹ Stop Realtime';
+                toggleBtn.style.background = '#ef4444';
+                toggleBtn.style.color = 'white';
+            }
+            captureAndSend();
+            realtimeTimer = setInterval(captureAndSend, 3000);
+            console.log('[EchoLens] Realtime ON');
+        } else {
+            isRunning = false;
+            clearInterval(realtimeTimer);
+            realtimeTimer = null;
+            if (toggleBtn) {
+                toggleBtn.textContent = '▶ Start Realtime';
+                toggleBtn.style.background = '';
+                toggleBtn.style.color = '';
+            }
+            console.log('[EchoLens] Realtime OFF');
+        }
+    };
+
+    window.echoDescribeOnce = function() {
+        captureAndSend();
+    };
+})();
 </script>
 """
 
 
 with gr.Blocks(title="EchoLens RT", theme=gr.themes.Soft()) as demo:
+
     gr.HTML(WEBCAM_JS)
-    gr.Markdown("# 👁️ EchoLens — Realtime Vision Assistant")
-    gr.Markdown("For blind and visually impaired users. Press **Start Realtime** to auto-describe every 3 seconds.")
 
-    is_realtime = gr.State(False)
+    gr.Markdown("""
+# 👁️ EchoLens — Realtime Vision Assistant
+**For blind and visually impaired users.**
+- Open your camera below
+- Press **Describe Once** for a single description
+- Press **Start Realtime** to auto-describe every 3 seconds
+""")
 
     with gr.Row():
+        # ── LEFT: camera ──────────────────────────────────────
         with gr.Column(scale=1):
             webcam_input = gr.Image(
                 label="Live Camera",
@@ -235,57 +253,58 @@ with gr.Blocks(title="EchoLens RT", theme=gr.themes.Soft()) as demo:
                 sources=["webcam"],
             )
             upload_input = gr.Image(
-                label="Or Upload Image",
+                label="Or Upload an Image",
                 type="numpy",
                 sources=["upload"],
             )
             task_choice = gr.Radio(
                 choices=["Quick (faster)", "Detailed (slower)"],
                 value="Quick (faster)",
-                label="Caption mode",
+                label="Caption detail",
             )
             with gr.Row():
-                btn = gr.Button("Describe Once ▶", variant="primary")
-                realtime_btn = gr.Button(
-                    "▶ Start Realtime",
-                    variant="secondary",
-                    elem_id="realtime-toggle-btn",
-                )
-
+                # These buttons call JS directly via elem_id
+                gr.HTML("""
+                <div style="display:flex; gap:8px; margin-top:4px;">
+                    <button
+                        onclick="window.echoDescribeOnce()"
+                        style="flex:1; padding:10px; background:#6366f1; color:white;
+                               border:none; border-radius:8px; font-size:15px; cursor:pointer;">
+                        📸 Describe Once
+                    </button>
+                    <button
+                        id="rt-btn-inner"
+                        onclick="window.echoToggleRealtime()"
+                        style="flex:1; padding:10px; background:#10b981; color:white;
+                               border:none; border-radius:8px; font-size:15px; cursor:pointer;">
+                        ▶ Start Realtime
+                    </button>
+                </div>
+                """)
+
+        # ── RIGHT: output ──────────────────────────────────────
         with gr.Column(scale=1):
             caption_out = gr.Textbox(
                 label="Caption",
-                lines=4,
+                lines=5,
                 interactive=False,
                 show_copy_button=True,
+                placeholder="Caption will appear here...",
             )
             audio_out = gr.Audio(
                 label="Audio Description",
                 type="filepath",
                 autoplay=True,
             )
-            gr.Markdown("*Realtime mode captures from webcam every 3 seconds.*")
+            gr.Markdown("*Realtime mode auto-describes every 3 seconds.*")
 
-    # Hidden components: JS writes frame here, triggers caption
+    # ── Hidden plumbing: JS → Python ──────────────────────────
     with gr.Row(visible=False):
-        frame_input = gr.Textbox(
-            elem_id="frame-input",
-            label="frame",
-        )
-        frame_btn = gr.Button(
-            "submit frame",
-            elem_id="frame-submit",
-        )
-
-    # Manual describe once (webcam snapshot)
-    btn.click(
-        fn=describe_upload,
-        inputs=[webcam_input, task_choice],
-        outputs=[caption_out, audio_out],
-        show_progress=False,
-    )
+        frame_box = gr.Textbox(elem_id="frame-box", label="frame_box")
+        with gr.Column(elem_id="frame-btn"):
+            frame_btn = gr.Button("send", elem_id="frame-submit")
 
-    # Upload image
+    # Upload describe
     upload_input.change(
         fn=describe_upload,
         inputs=[upload_input, task_choice],
@@ -293,37 +312,13 @@ with gr.Blocks(title="EchoLens RT", theme=gr.themes.Soft()) as demo:
         show_progress=False,
     )
 
-    # Hidden frame button → caption
+    # Hidden frame button → describe_frame (realtime + describe-once via JS)
     frame_btn.click(
         fn=describe_frame,
-        inputs=[frame_input, task_choice],
+        inputs=[frame_box, task_choice],
         outputs=[caption_out, audio_out],
         show_progress=False,
-    )
-
-    # Realtime toggle: update button label + call JS start/stop
-    realtime_btn.click(
-        fn=lambda s: (
-            not s,
-            gr.update(
-                value="⏹ Stop Realtime" if not s else "▶ Start Realtime",
-                variant="stop" if not s else "secondary",
-            ),
-        ),
-        inputs=[is_realtime],
-        outputs=[is_realtime, realtime_btn],
-    ).then(
-        fn=None,
-        js="""
-        (is_realtime) => {
-            if (is_realtime) {
-                window.startRealtime();
-            } else {
-                window.stopRealtime();
-            }
-        }
-        """,
-        inputs=[is_realtime],
+        queue=True,
     )
 
 if __name__ == "__main__":