File size: 5,248 Bytes
e1c010e
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
"""
src/ui/handlers.py — Gradio event-handler functions for the VLA pipeline.

These functions are wired to Gradio UI events (button clicks, etc.) and
are intentionally kept free of Gradio component construction so they can
be unit-tested independently.
"""

import json
import logging

import cv2
import numpy as np

from src.pipeline.vla_pipeline import VLAPipeline

logger = logging.getLogger(__name__)

# ---------------------------------------------------------------------------
# Shared pipeline instance (initialised once at import time)
# ---------------------------------------------------------------------------
_pipeline: VLAPipeline | None = None


def get_pipeline() -> VLAPipeline:
    """Return the shared VLAPipeline, initialising it on first call."""
    global _pipeline
    if _pipeline is None:
        logger.info("Initializing VLA Pipeline...")
        _pipeline = VLAPipeline()
        logger.info("VLA Pipeline ready.")
    return _pipeline


# ---------------------------------------------------------------------------
# Main Gradio handler
# ---------------------------------------------------------------------------
def process_pipeline(
    image: np.ndarray | None,
    command: str,
) -> tuple:
    """
    Run the VLA pipeline and return outputs for Gradio components.

    Args:
        image:   Input image as a NumPy array (from gr.Image).
        command: Natural-language instruction from the user.

    Returns:
        5-tuple:
            - detection_image (np.ndarray)
            - detections_json (str)
            - action_plan_md  (str)
            - action_image    (np.ndarray)
            - status_message  (str)
    """
    empty = _blank_image("Upload an image and\nenter a command to begin")

    if image is None:
        return empty, "[]", "*Please upload an image.*", empty, "⚠️ No image provided."

    if not command or not command.strip():
        return empty, "[]", "*Please enter a command.*", empty, "⚠️ No command entered."

    logger.info("Processing: '%s'", command)
    result = get_pipeline().run(image, command)

    detections_json = json.dumps(result.detections, indent=2) if result.detections else "[]"
    action_plan_md = _format_action_plan(result.action_plan, result.interpretation)

    # Status bar
    if result.error:
        status = f"⚠️ {result.error}"
    else:
        plan_status = result.action_plan.get("status", "unknown")
        emoji = {"success": "✅", "partial": "⚠️", "failed": "❌"}.get(plan_status, "❓")
        n = len(result.detections)
        steps = len(result.action_plan.get("steps", []))
        status = (
            f"{emoji} Pipeline complete — Plan status: **{plan_status}**"
            f" |  {n} object{'s' if n != 1 else ''} detected"
            f" |  {steps} action step{'s' if steps != 1 else ''} generated"
        )

    det_img = _ensure_rgb(result.detection_image, empty)
    act_img = _ensure_rgb(result.action_image, empty)
    return det_img, detections_json, action_plan_md, act_img, status


# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _format_action_plan(action_plan: dict, interpretation: dict) -> str:
    """Format the action plan dict as a readable Markdown string."""
    if not action_plan:
        return "*No action plan generated.*"

    lines: list[str] = []

    if interpretation:
        lines += [
            "### 🧠 Command Interpretation",
            "| Field | Value |",
            "|-------|-------|",
            f"| **Action** | `{interpretation.get('action', 'N/A')}` |",
            f"| **Source** | `{interpretation.get('source', 'N/A')}` |",
            f"| **Destination** | `{interpretation.get('destination') or 'N/A'}` |",
            "",
        ]

    steps = action_plan.get("steps", [])
    if steps:
        lines.append("### 📋 Execution Steps")
        for i, step in enumerate(steps, 1):
            lines.append(f"**{i}.** {step}")
        lines.append("")

    status = action_plan.get("status", "unknown")
    emoji = {"success": "✅", "partial": "⚠️", "failed": "❌"}.get(status, "❓")
    lines += ["---", f"### {emoji} Status: `{status.upper()}`"]

    return "\n".join(lines)


def _ensure_rgb(image: np.ndarray | None, fallback: np.ndarray) -> np.ndarray:
    """Convert image to RGB format expected by Gradio."""
    if image is None:
        return fallback
    if len(image.shape) == 2:
        return cv2.cvtColor(image, cv2.COLOR_GRAY2RGB)
    if image.shape[2] == 4:
        return cv2.cvtColor(image, cv2.COLOR_BGRA2RGB)
    return image


def _blank_image(text: str = "") -> np.ndarray:
    """Return a dark placeholder image with optional centred text."""
    img = np.full((400, 600, 3), 18, dtype=np.uint8)  # near-black
    if text:
        for i, line in enumerate(text.split("\n")):
            y = 185 + i * 34
            (w, _), _ = cv2.getTextSize(line, cv2.FONT_HERSHEY_SIMPLEX, 0.7, 1)
            x = max(0, (600 - w) // 2)
            cv2.putText(img, line, (x, y), cv2.FONT_HERSHEY_SIMPLEX,
                        0.7, (100, 100, 100), 1, cv2.LINE_AA)
    return img