File size: 5,248 Bytes
e1c010e | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 | """
src/ui/handlers.py — Gradio event-handler functions for the VLA pipeline.
These functions are wired to Gradio UI events (button clicks, etc.) and
are intentionally kept free of Gradio component construction so they can
be unit-tested independently.
"""
import json
import logging
import cv2
import numpy as np
from src.pipeline.vla_pipeline import VLAPipeline
logger = logging.getLogger(__name__)
# ---------------------------------------------------------------------------
# Shared pipeline instance (initialised once at import time)
# ---------------------------------------------------------------------------
_pipeline: VLAPipeline | None = None
def get_pipeline() -> VLAPipeline:
"""Return the shared VLAPipeline, initialising it on first call."""
global _pipeline
if _pipeline is None:
logger.info("Initializing VLA Pipeline...")
_pipeline = VLAPipeline()
logger.info("VLA Pipeline ready.")
return _pipeline
# ---------------------------------------------------------------------------
# Main Gradio handler
# ---------------------------------------------------------------------------
def process_pipeline(
image: np.ndarray | None,
command: str,
) -> tuple:
"""
Run the VLA pipeline and return outputs for Gradio components.
Args:
image: Input image as a NumPy array (from gr.Image).
command: Natural-language instruction from the user.
Returns:
5-tuple:
- detection_image (np.ndarray)
- detections_json (str)
- action_plan_md (str)
- action_image (np.ndarray)
- status_message (str)
"""
empty = _blank_image("Upload an image and\nenter a command to begin")
if image is None:
return empty, "[]", "*Please upload an image.*", empty, "⚠️ No image provided."
if not command or not command.strip():
return empty, "[]", "*Please enter a command.*", empty, "⚠️ No command entered."
logger.info("Processing: '%s'", command)
result = get_pipeline().run(image, command)
detections_json = json.dumps(result.detections, indent=2) if result.detections else "[]"
action_plan_md = _format_action_plan(result.action_plan, result.interpretation)
# Status bar
if result.error:
status = f"⚠️ {result.error}"
else:
plan_status = result.action_plan.get("status", "unknown")
emoji = {"success": "✅", "partial": "⚠️", "failed": "❌"}.get(plan_status, "❓")
n = len(result.detections)
steps = len(result.action_plan.get("steps", []))
status = (
f"{emoji} Pipeline complete — Plan status: **{plan_status}**"
f" | {n} object{'s' if n != 1 else ''} detected"
f" | {steps} action step{'s' if steps != 1 else ''} generated"
)
det_img = _ensure_rgb(result.detection_image, empty)
act_img = _ensure_rgb(result.action_image, empty)
return det_img, detections_json, action_plan_md, act_img, status
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _format_action_plan(action_plan: dict, interpretation: dict) -> str:
"""Format the action plan dict as a readable Markdown string."""
if not action_plan:
return "*No action plan generated.*"
lines: list[str] = []
if interpretation:
lines += [
"### 🧠 Command Interpretation",
"| Field | Value |",
"|-------|-------|",
f"| **Action** | `{interpretation.get('action', 'N/A')}` |",
f"| **Source** | `{interpretation.get('source', 'N/A')}` |",
f"| **Destination** | `{interpretation.get('destination') or 'N/A'}` |",
"",
]
steps = action_plan.get("steps", [])
if steps:
lines.append("### 📋 Execution Steps")
for i, step in enumerate(steps, 1):
lines.append(f"**{i}.** {step}")
lines.append("")
status = action_plan.get("status", "unknown")
emoji = {"success": "✅", "partial": "⚠️", "failed": "❌"}.get(status, "❓")
lines += ["---", f"### {emoji} Status: `{status.upper()}`"]
return "\n".join(lines)
def _ensure_rgb(image: np.ndarray | None, fallback: np.ndarray) -> np.ndarray:
"""Convert image to RGB format expected by Gradio."""
if image is None:
return fallback
if len(image.shape) == 2:
return cv2.cvtColor(image, cv2.COLOR_GRAY2RGB)
if image.shape[2] == 4:
return cv2.cvtColor(image, cv2.COLOR_BGRA2RGB)
return image
def _blank_image(text: str = "") -> np.ndarray:
"""Return a dark placeholder image with optional centred text."""
img = np.full((400, 600, 3), 18, dtype=np.uint8) # near-black
if text:
for i, line in enumerate(text.split("\n")):
y = 185 + i * 34
(w, _), _ = cv2.getTextSize(line, cv2.FONT_HERSHEY_SIMPLEX, 0.7, 1)
x = max(0, (600 - w) // 2)
cv2.putText(img, line, (x, y), cv2.FONT_HERSHEY_SIMPLEX,
0.7, (100, 100, 100), 1, cv2.LINE_AA)
return img
|