Merge branch 'master'

.env.example

@@ -0,0 +1,31 @@
+# Environment Configuration Template
+# Copy this file to .env and fill in your values.
+
+# ── LLM Provider ──────────────────────────────────────────────────────────────
+# Choose: "openai" or "groq"
+# Groq is free-tier and ~5-10x faster — recommended for HF Spaces
+LLM_PROVIDER="groq"
+
+# OpenAI  (used when LLM_PROVIDER=openai)
+OPENAI_API_KEY=""
+OPENAI_MODEL="gpt-4o-mini"
+
+# Groq  (used when LLM_PROVIDER=groq) — get key at console.groq.com
+GROQ_API_KEY=""
+GROQ_MODEL="llama-3.3-70b-versatile"
+
+# ── Vision / Detection ────────────────────────────────────────────────────────
+# Choose: "grounding_dino"  (open-vocabulary, recommended)
+#      or "yolo"            (COCO-80 fixed classes, faster)
+DETECTION_MODEL="grounding_dino"
+
+# Grounding DINO settings
+GROUNDING_DINO_MODEL="IDEA-Research/grounding-dino-tiny"
+GROUNDING_DINO_CONFIDENCE="0.3"
+
+# YOLO settings (only used when DETECTION_MODEL=yolo)
+YOLO_MODEL="yolov8n.pt"
+YOLO_CONFIDENCE="0.25"
+
+# ── Logging ───────────────────────────────────────────────────────────────────
+LOG_LEVEL="INFO"

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+*.jpg filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,40 @@
+# Environment
+.env
+
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+*.egg-info/
+dist/
+build/
+*.egg
+
+# YOLO model weights
+*.pt
+models/
+
+# Virtual environment
+.venv/
+venv/
+env/
+
+# IDE
+.vscode/
+.idea/
+*.swp
+*.swo
+
+# OS
+.DS_Store
+Thumbs.db
+
+# Logs
+*.log
+
+# Gradio
+flagged/
+
+README_APP.md

README.md

@@ -1,14 +1,53 @@
 ---
-title: Codevally Ai
-emoji: 🦀
-colorFrom: green
-colorTo: pink
+title: VLA Demo - Codevally AI
+emoji: 🤖
+colorFrom: blue
+colorTo: indigo
 sdk: gradio
-sdk_version: 6.9.0
+sdk_version: "4.44.0"
 app_file: app.py
 pinned: false
-license: mit
-short_description: Codevally Portfolios
+license: other
+short_description: Vision Language Action pipeline — open-vocabulary AI demo
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# 🤖 Vision Language Action (VLA) Demo
+
+> Built by **[Codevally](https://www.codevally.com)** — AI-Powered Software Development & AI Solutions
+
+Upload an image and enter a natural-language command. The system detects objects using **Grounding DINO** (open-vocabulary), interprets your instruction with an **LLM**, generates a structured action plan, and visualizes the result.
+
+**Try it:** _"Pick up the defective board and place it in the red bin"_
+
+## How it works
+
+1. 📸 **Vision** — Grounding DINO detects objects based on your command text
+2. 🧠 **Language** — LLM (OpenAI / Groq) interprets the command
+3. 📋 **Planning** — Structured action steps are generated
+4. 🎬 **Action** — Annotated visualization is produced
+
+---
+
+## 🏢 About Codevally
+
+[Codevally](https://www.codevally.com) is a forward-thinking technology company at the intersection of **AI and software development**. We specialize in building intelligent systems, AI-powered applications, and custom AI solutions that transform businesses — from traditional software to cutting-edge machine learning implementations.
+
+### 🔬 Our AI Specializations
+
+| Domain                        | Capabilities                                                           |
+| ----------------------------- | ---------------------------------------------------------------------- |
+| 🖼️ **Computer Vision**        | Object detection, image classification, video analytics, visual search |
+| 🤖 **Language Models (LLMs)** | OpenAI, Claude, Gemini, Llama fine-tuning, RAG systems, AI agents      |
+| 💬 **NLP**                    | Sentiment analysis, NER, text summarization, question answering        |
+| 📈 **Predictive AI**          | Forecasting, anomaly detection, churn & risk scoring                   |
+
+### ✅ Why Codevally
+
+- **AI-First Approach** — Deep expertise in AI/ML and modern LLM technologies
+- **End-to-End Solutions** — From concept to deployment and monitoring (MLOps)
+- **Practical AI** — Focused on real business value, not just technology
+- **Responsible AI** — Ethics, bias detection, and governance built-in
+
+---
+
+_This demo is an open-vocabulary Vision Language Action pipeline built by the Codevally AI team. [Learn more →](https://www.codevally.com)_

app.py

@@ -0,0 +1,44 @@
+"""
+app.py — VLA Demo entry point.
+
+Compatible with:
+  - Local:      python app.py
+  - HF Spaces:  imported directly (demo object must exist at module level)
+
+All UI logic lives in the src/ui/ package.
+"""
+
+import logging
+
+from src.utils.config import (
+    setup_logging, validate_config,
+    LLM_PROVIDER, OPENAI_MODEL, GROQ_MODEL,
+    DETECTION_MODEL,
+)
+from src.ui import create_ui
+
+setup_logging()
+logger = logging.getLogger(__name__)
+
+for warning in validate_config():
+    logger.warning(warning)
+
+# ── Startup config summary ─────────────────────────────────────────────────
+_active_model = GROQ_MODEL if LLM_PROVIDER == "groq" else OPENAI_MODEL
+logger.info("=" * 60)
+logger.info("  Codevally VLA Demo — startup configuration")
+logger.info("  LLM Provider  : %s", LLM_PROVIDER.upper())
+logger.info("  LLM Model     : %s", _active_model)
+logger.info("  Vision Model  : %s", DETECTION_MODEL)
+logger.info("=" * 60)
+
+# Build the Gradio app at module level so HF Spaces can find it
+demo = create_ui()
+
+if __name__ == "__main__":
+    logger.info("Launching Codevally VLA Demo...")
+    demo.launch(
+        server_name="0.0.0.0",
+        server_port=7860,
+        share=False,
+    )

requirements.txt

@@ -0,0 +1,16 @@
+# Install CPU-only PyTorch first
+--extra-index-url https://download.pytorch.org/whl/cpu
+
+torch
+torchvision
+torchaudio
+
+ultralytics>=8.0.0
+opencv-python>=4.8.0
+gradio>=4.0.0
+openai>=1.0.0
+numpy>=1.24.0,<2.0.0
+Pillow>=9.0.0
+python-dotenv>=1.0.0
+transformers>=4.40.0
+pytest>=7.0.0

src/__init__.py

@@ -0,0 +1 @@
+# VLA Demo System - Source Package

src/language/__init__.py

@@ -0,0 +1 @@
+# Language Module Package

src/language/reasoner.py

@@ -0,0 +1,329 @@
+"""
+Language Reasoning Module — Natural-language command interpretation via LLM.
+
+Takes a user command and the list of detected objects, then asks an LLM
+to produce a structured action interpretation:
+
+    {"action": "move", "source": "bottle", "destination": "box"}
+
+Supported providers (set via LLM_PROVIDER env var):
+    - "openai"  → OpenAI API  (GPT-4o-mini by default)
+    - "groq"    → Groq API    (Llama 3.3 70B by default, ~5-10x faster)
+"""
+
+import json
+import logging
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+# ---------------------------------------------------------------------------
+# System prompt sent to the LLM
+# ---------------------------------------------------------------------------
+_SYSTEM_PROMPT = """\
+You are an AI assistant for a Vision Language Action robotics system.
+
+Given:
+  1. A list of objects detected in a scene (with labels and bounding boxes).
+  2. A natural-language command from a human operator.
+
+Your task:
+  Interpret the command and return a JSON object with exactly these keys:
+    - "action"      : The verb describing what to do (e.g. "move", "pick_up",
+                       "place", "inspect", "rotate", "remove").
+    - "source"      : The object to act on.
+    - "destination"  : The target location or object (if applicable, else null).
+
+CRITICAL RULES:
+  • For "source" and "destination", you MUST use the EXACT label string from
+    the detected objects list. The user may refer to objects using different
+    words (e.g. "wrench" might actually be detected as "knife", "toolbox"
+    might be "suitcase", "mug" might be "cup"). Your job is to figure out
+    which detected object the user is referring to and use that detected label.
+  • If the user says "wrench" but only "knife" is detected, return "knife".
+  • If the user says "toolbox" but only "suitcase" is detected, return "suitcase".
+  • If the user says "bin" but only "bowl" is detected, return "bowl".
+  • NEVER return a label that is not in the detected objects list.
+  • If truly no detected object can reasonably match, use the closest detected
+    object label and note it.
+  • Return ONLY the JSON object, no extra text.
+  • If the command is ambiguous, make your best interpretation.
+  • If no destination is implied, set "destination" to null.
+"""
+
+
+# Provider → base URL mapping (Groq uses the OpenAI-compatible SDK)
+_PROVIDER_BASE_URLS: dict[str, str | None] = {
+    "openai": None,                           # uses default OpenAI endpoint
+    "groq":   "https://api.groq.com/openai/v1",
+}
+
+
+class CommandReasoner:
+    """
+    Interprets natural-language commands using an OpenAI-compatible LLM.
+
+    Supports OpenAI and Groq as backends — both expose the same
+    Chat Completions API so the same prompt and parsing logic work for both.
+
+    Attributes:
+        provider: Active provider name ("openai" or "groq").
+        model:    Model identifier (e.g. "gpt-4o-mini" or "llama-3.3-70b-versatile").
+        client:   OpenAI SDK client instance (or None in fallback mode).
+    """
+
+    def __init__(
+        self,
+        provider: str,
+        api_key: str,
+        model: str,
+    ):
+        """
+        Initialise the reasoner for a specific provider.
+
+        Args:
+            provider: "openai" or "groq".
+            api_key:  API key for the chosen provider.
+            model:    Model identifier string.
+        """
+        self.provider = provider
+        self.model = model
+        self.client = None
+
+        if api_key:
+            self._init_client(provider, api_key)
+        else:
+            logger.warning(
+                "No API key provided for provider='%s' — "
+                "CommandReasoner will use fallback mode.",
+                provider,
+            )
+
+    def _init_client(self, provider: str, api_key: str) -> None:
+        """Create an OpenAI-SDK client pointed at the chosen provider endpoint."""
+        try:
+            from openai import OpenAI
+
+            base_url = _PROVIDER_BASE_URLS.get(provider)  # None → OpenAI default
+            kwargs: dict = {"api_key": api_key}
+            if base_url:
+                kwargs["base_url"] = base_url
+
+            self.client = OpenAI(**kwargs)
+            logger.info(
+                "LLM client initialised (provider=%s, model=%s).",
+                provider,
+                self.model,
+            )
+        except Exception as exc:
+            logger.error("Failed to initialise LLM client (%s): %s", provider, exc)
+            self.client = None
+
+    # ------------------------------------------------------------------
+    # Public API
+    # ------------------------------------------------------------------
+
+    def interpret(
+        self,
+        command: str,
+        detected_objects: list[dict[str, Any]],
+    ) -> dict[str, Any]:
+        """
+        Interpret a user command in the context of detected objects.
+
+        Args:
+            command: Free-form user instruction.
+            detected_objects: List of detection dicts from the vision module.
+
+        Returns:
+            Dict with keys ``action``, ``source``, ``destination``.
+        """
+        if not command or not command.strip():
+            logger.warning("Empty command received.")
+            return self._error_result("No command provided.")
+
+        if self.client is None:
+            logger.info("Using fallback reasoning (no LLM client).")
+            return self._fallback_interpret(command, detected_objects)
+
+        return self._llm_interpret(command, detected_objects)
+
+    # ------------------------------------------------------------------
+    # LLM-based interpretation
+    # ------------------------------------------------------------------
+
+    def _llm_interpret(
+        self,
+        command: str,
+        detected_objects: list[dict[str, Any]],
+    ) -> dict[str, Any]:
+        """Call the LLM to interpret the command."""
+        objects_description = self._format_objects(detected_objects)
+
+        user_message = (
+            f"Detected objects in the scene:\n{objects_description}\n\n"
+            f"User command: \"{command}\"\n\n"
+            f"Return the structured JSON interpretation."
+        )
+
+        try:
+            response = self.client.chat.completions.create(
+                model=self.model,
+                messages=[
+                    {"role": "system", "content": _SYSTEM_PROMPT},
+                    {"role": "user", "content": user_message},
+                ],
+                temperature=0.1,
+                max_tokens=256,
+            )
+
+            raw = response.choices[0].message.content.strip()
+            logger.info("LLM raw response: %s", raw)
+
+            return self._parse_response(raw)
+
+        except Exception as exc:
+            logger.error("LLM call failed: %s", exc)
+            return self._fallback_interpret(command, detected_objects)
+
+    # ------------------------------------------------------------------
+    # Fallback (no LLM)
+    # ------------------------------------------------------------------
+
+    def _fallback_interpret(
+        self,
+        command: str,
+        detected_objects: list[dict[str, Any]],
+    ) -> dict[str, Any]:
+        """
+        Smart keyword-based fallback when no LLM is available.
+
+        Uses the ObjectMapper for synonym-aware matching between user
+        terms and detected YOLO labels.
+        """
+        from src.utils.object_mapper import find_best_match
+
+        logger.info("Fallback interpretation for: '%s'", command)
+
+        # Try to determine action verb
+        action = "move"
+        for verb in ("pick", "place", "move", "inspect", "rotate", "remove", "find"):
+            if verb in command.lower():
+                action = verb
+                break
+
+        labels = [obj["label"] for obj in detected_objects]
+        cmd_lower = command.lower()
+
+        # Extract noun phrases from command by removing common verbs/prepositions
+        stop_words = {
+            "the", "a", "an", "and", "or", "to", "from", "in", "on", "at",
+            "into", "onto", "of", "with", "for", "it", "up", "back", "next",
+            "pick", "place", "move", "inspect", "rotate", "remove", "find",
+            "put", "take", "grab", "get", "bring", "carry", "push", "pull",
+            "near", "close", "closest", "nearest", "under", "over", "above",
+            "below", "beside", "behind",
+        }
+        words = cmd_lower.replace(",", " ").replace(".", " ").split()
+
+        # Build candidate noun phrases (single words + bigrams)
+        candidates: list[str] = []
+        for w in words:
+            if w not in stop_words and len(w) > 1:
+                candidates.append(w)
+        for i in range(len(words) - 1):
+            bigram = f"{words[i]} {words[i+1]}"
+            if words[i] not in stop_words or words[i+1] not in stop_words:
+                candidates.append(bigram)
+
+        # Match candidates against detected labels using ObjectMapper
+        source = None
+        destination = None
+        used_labels: set[str] = set()
+
+        for candidate in candidates:
+            match = find_best_match(candidate, labels)
+            if match and match not in used_labels:
+                if source is None:
+                    source = match
+                    used_labels.add(match)
+                elif destination is None:
+                    destination = match
+                    used_labels.add(match)
+                    break
+
+        # Fallback: if no matches, use first two detected objects
+        if source is None and labels:
+            source = labels[0]
+        if destination is None and len(labels) > 1:
+            remaining = [l for l in labels if l != source]
+            if remaining:
+                destination = remaining[0]
+
+        return {
+            "action": action,
+            "source": source or "unknown",
+            "destination": destination,
+        }
+
+    # ------------------------------------------------------------------
+    # Helpers
+    # ------------------------------------------------------------------
+
+    @staticmethod
+    def _format_objects(detected_objects: list[dict[str, Any]]) -> str:
+        """Format detected objects as a readable string for the LLM prompt."""
+        if not detected_objects:
+            return "No objects detected."
+
+        lines: list[str] = []
+        for i, obj in enumerate(detected_objects, 1):
+            bbox = obj.get("bbox", [])
+            conf = obj.get("confidence", 0)
+            lines.append(
+                f"  {i}. {obj['label']} (confidence: {conf:.2f}, bbox: {bbox})"
+            )
+        return "\n".join(lines)
+
+    @staticmethod
+    def _parse_response(raw: str) -> dict[str, Any]:
+        """
+        Parse the LLM response string into a dict.
+
+        Handles markdown code fences and extra whitespace.
+        """
+        cleaned = raw.strip()
+        # Strip optional markdown code fences
+        if cleaned.startswith("```"):
+            lines = cleaned.split("\n")
+            lines = [ln for ln in lines if not ln.strip().startswith("```")]
+            cleaned = "\n".join(lines).strip()
+
+        try:
+            result = json.loads(cleaned)
+        except json.JSONDecodeError:
+            logger.warning("Failed to parse LLM JSON: %s", cleaned)
+            return {
+                "action": "unknown",
+                "source": "unknown",
+                "destination": None,
+                "error": "Failed to parse LLM response.",
+                "raw_response": raw,
+            }
+
+        # Ensure required keys
+        return {
+            "action": result.get("action", "unknown"),
+            "source": result.get("source", "unknown"),
+            "destination": result.get("destination"),
+        }
+
+    @staticmethod
+    def _error_result(message: str) -> dict[str, Any]:
+        """Return a structured error result."""
+        return {
+            "action": "error",
+            "source": "unknown",
+            "destination": None,
+            "error": message,
+        }

src/pipeline/__init__.py

@@ -0,0 +1 @@
+# Pipeline Module Package

src/pipeline/vla_pipeline.py

@@ -0,0 +1,220 @@
+"""
+Pipeline Orchestrator — Wires all VLA modules together into a single
+``run()`` call.
+
+Supports two detection backends (set via ``DETECTION_MODEL`` env var):
+  - ``"yolo"``          — YOLOv8, fixed 80 COCO classes
+  - ``"grounding_dino"`` — Open-vocabulary, detects any text-prompted object
+
+The ``VLAPipeline`` class initializes the selected detector, reasoner,
+planner, and visualizer, then exposes one public method that drives the
+full Vision → Language → Action flow.
+"""
+
+import logging
+from dataclasses import dataclass, field
+from typing import Any
+
+import numpy as np
+
+from src.utils.config import (
+    DETECTION_MODEL,
+    LLM_PROVIDER,
+    OPENAI_API_KEY,
+    OPENAI_MODEL,
+    GROQ_API_KEY,
+    GROQ_MODEL,
+    YOLO_CONFIDENCE,
+    YOLO_MODEL,
+    GROUNDING_DINO_MODEL,
+    GROUNDING_DINO_CONFIDENCE,
+)
+from src.language.reasoner import CommandReasoner
+from src.planner.action_planner import ActionPlanner
+from src.visualization.visualizer import ActionVisualizer
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class PipelineResult:
+    """Container for all intermediate and final pipeline outputs."""
+
+    detections: list[dict[str, Any]] = field(default_factory=list)
+    interpretation: dict[str, Any] = field(default_factory=dict)
+    action_plan: dict[str, Any] = field(default_factory=dict)
+    detection_image: np.ndarray | None = None
+    action_image: np.ndarray | None = None
+    error: str | None = None
+
+
+class VLAPipeline:
+    """
+    Orchestrates the full Vision Language Action pipeline.
+
+    The detector backend is chosen via the ``DETECTION_MODEL`` environment
+    variable:
+
+        - ``"yolo"``           → :class:`ObjectDetector` (COCO-80)
+        - ``"grounding_dino"`` → :class:`GroundingDINODetector` (open vocabulary)
+
+    Usage::
+
+        pipeline = VLAPipeline()
+        result = pipeline.run(image, "Move the bottle to the box")
+    """
+
+    def __init__(self) -> None:
+        """Initialize all sub-modules."""
+        logger.info("Initializing VLA Pipeline...")
+
+        # Select detection backend
+        self.detection_mode = DETECTION_MODEL.lower()
+        self.detector: Any = None
+        self._grounding_dino = None  # Only set when using Grounding DINO
+
+        if self.detection_mode == "grounding_dino":
+            self._init_grounding_dino()
+        else:
+            self._init_yolo()
+
+        # Select LLM provider
+        api_key = GROQ_API_KEY if LLM_PROVIDER == "groq" else OPENAI_API_KEY
+        model   = GROQ_MODEL   if LLM_PROVIDER == "groq" else OPENAI_MODEL
+        self.reasoner = CommandReasoner(
+            provider=LLM_PROVIDER,
+            api_key=api_key,
+            model=model,
+        )
+        self.planner = ActionPlanner()
+        self.visualizer = ActionVisualizer()
+
+        logger.info(
+            "VLA Pipeline initialized (detector=%s, llm_provider=%s, model=%s).",
+            self.detection_mode,
+            LLM_PROVIDER,
+            model,
+        )
+
+    def _init_yolo(self) -> None:
+        """Initialize the YOLO detector."""
+        from src.vision.detector import ObjectDetector
+
+        self.detector = ObjectDetector(
+            model_name=YOLO_MODEL,
+            confidence=YOLO_CONFIDENCE,
+        )
+
+    def _init_grounding_dino(self) -> None:
+        """Initialize the Grounding DINO detector."""
+        from src.vision.grounding_dino_detector import GroundingDINODetector
+
+        gd = GroundingDINODetector(
+            model_name=GROUNDING_DINO_MODEL,
+            confidence=GROUNDING_DINO_CONFIDENCE,
+        )
+        self.detector = gd
+        self._grounding_dino = gd
+
+    def run(self, image: np.ndarray, command: str) -> PipelineResult:
+        """
+        Execute the full VLA pipeline.
+
+        Args:
+            image: Input image as a NumPy array (H, W, C).
+            command: Natural-language user command.
+
+        Returns:
+            ``PipelineResult`` containing all intermediate outputs and
+            the final annotated images.
+        """
+        result = PipelineResult()
+
+        # Validate inputs
+        if image is None or not isinstance(image, np.ndarray):
+            result.error = "Invalid image provided."
+            logger.error(result.error)
+            return result
+
+        if not command or not command.strip():
+            result.error = "No command provided."
+            logger.error(result.error)
+            return result
+
+        logger.info("=== VLA Pipeline Start ===")
+        logger.info("Command: '%s'", command)
+        logger.info("Detector: %s", self.detection_mode)
+
+        # ------------------------------------------------------------------
+        # Step 1: Vision — Detect objects
+        # ------------------------------------------------------------------
+        logger.info("[1/4] Running object detection...")
+        try:
+            if self._grounding_dino is not None:
+                # Build a text prompt from the user's command
+                from src.vision.grounding_dino_detector import GroundingDINODetector
+
+                text_prompt = GroundingDINODetector.build_prompt_from_command(command)
+                result.detections = self._grounding_dino.detect(image, text_prompt)
+            else:
+                result.detections = self.detector.detect(image)
+
+            logger.info("Detected %d objects.", len(result.detections))
+        except Exception as exc:
+            result.error = f"Detection failed: {exc}"
+            logger.error(result.error)
+            return result
+
+        # Generate detection overlay image
+        result.detection_image = self.visualizer.draw_detections(
+            image, result.detections
+        )
+
+        if not result.detections:
+            result.error = "No objects detected in the image."
+            logger.warning(result.error)
+            return result
+
+        # ------------------------------------------------------------------
+        # Step 2: Language — Interpret command
+        # ------------------------------------------------------------------
+        logger.info("[2/4] Interpreting command...")
+        try:
+            result.interpretation = self.reasoner.interpret(
+                command, result.detections
+            )
+            logger.info("Interpretation: %s", result.interpretation)
+        except Exception as exc:
+            result.error = f"Command interpretation failed: {exc}"
+            logger.error(result.error)
+            return result
+
+        # ------------------------------------------------------------------
+        # Step 3: Planning — Generate action plan
+        # ------------------------------------------------------------------
+        logger.info("[3/4] Generating action plan...")
+        try:
+            result.action_plan = self.planner.plan(
+                result.detections, result.interpretation
+            )
+            logger.info("Plan status: %s", result.action_plan.get("status"))
+        except Exception as exc:
+            result.error = f"Action planning failed: {exc}"
+            logger.error(result.error)
+            return result
+
+        # ------------------------------------------------------------------
+        # Step 4: Visualization — Render action overlay
+        # ------------------------------------------------------------------
+        logger.info("[4/4] Generating action visualization...")
+        try:
+            result.action_image = self.visualizer.visualize(
+                image, result.action_plan
+            )
+        except Exception as exc:
+            result.error = f"Visualization failed: {exc}"
+            logger.error(result.error)
+            return result
+
+        logger.info("=== VLA Pipeline Complete ===")
+        return result

src/planner/__init__.py

@@ -0,0 +1 @@
+# Planner Module Package

src/planner/action_planner.py

@@ -0,0 +1,197 @@
+"""
+Action Planning Module — Generates structured action steps from
+detected objects and the interpreted command.
+
+Uses the ObjectMapper for fuzzy label matching between the command
+interpretation and the actual YOLO detections.
+"""
+
+import logging
+from typing import Any
+
+from src.utils.object_mapper import find_best_match
+
+logger = logging.getLogger(__name__)
+
+
+class ActionPlanner:
+    """
+    Generates a step-by-step action plan by matching the interpreted
+    command to detected objects in the scene.
+    """
+
+    def plan(
+        self,
+        detected_objects: list[dict[str, Any]],
+        command_interpretation: dict[str, Any],
+    ) -> dict[str, Any]:
+        """
+        Create an action plan from detections and the interpreted command.
+
+        Args:
+            detected_objects: Objects detected by the vision module.
+            command_interpretation: Output from the language reasoner
+                (keys: ``action``, ``source``, ``destination``).
+
+        Returns:
+            Dict containing:
+                - ``steps`` (list[str]): Human-readable action steps.
+                - ``source_object`` (dict | None): Matched source detection.
+                - ``dest_object`` (dict | None): Matched destination detection.
+                - ``source_bbox`` (list | None): Source bounding box.
+                - ``dest_bbox`` (list | None): Destination bounding box.
+                - ``action`` (str): The action verb.
+                - ``status`` (str): ``"success"`` or ``"partial"`` or ``"failed"``.
+        """
+        if not detected_objects:
+            logger.warning("No detected objects provided to planner.")
+            return self._empty_plan("No objects detected in the scene.")
+
+        action = command_interpretation.get("action", "unknown")
+        source_query = command_interpretation.get("source", "")
+        dest_query = command_interpretation.get("destination")
+
+        detected_labels = [obj["label"] for obj in detected_objects]
+
+        # Match source object
+        source_obj = self._match_object(source_query, detected_labels, detected_objects)
+        # Match destination object (optional)
+        dest_obj = (
+            self._match_object(dest_query, detected_labels, detected_objects)
+            if dest_query
+            else None
+        )
+
+        # Build the step list
+        steps = self._build_steps(action, source_query, dest_query, source_obj, dest_obj)
+
+        # Determine status
+        status = self._determine_status(source_obj, dest_obj, dest_query)
+
+        plan = {
+            "action": action,
+            "steps": steps,
+            "source_object": source_obj,
+            "dest_object": dest_obj,
+            "source_bbox": source_obj["bbox"] if source_obj else None,
+            "dest_bbox": dest_obj["bbox"] if dest_obj else None,
+            "status": status,
+        }
+
+        logger.info(
+            "Action plan generated: %d steps, status=%s", len(steps), status
+        )
+        return plan
+
+    # ------------------------------------------------------------------
+    # Internal helpers
+    # ------------------------------------------------------------------
+
+    @staticmethod
+    def _match_object(
+        query: str | None,
+        detected_labels: list[str],
+        detected_objects: list[dict[str, Any]],
+    ) -> dict[str, Any] | None:
+        """
+        Match a query string to a detected object using fuzzy matching.
+
+        Returns the full detection dict or None.
+        """
+        if not query:
+            return None
+
+        matched_label = find_best_match(query, detected_labels)
+        if matched_label is None:
+            logger.warning("Could not match '%s' to any detected object.", query)
+            return None
+
+        # Return the first detection with this label
+        for obj in detected_objects:
+            if obj["label"] == matched_label:
+                return obj
+
+        return None
+
+    @staticmethod
+    def _build_steps(
+        action: str,
+        source_query: str,
+        dest_query: str | None,
+        source_obj: dict | None,
+        dest_obj: dict | None,
+    ) -> list[str]:
+        """Build human-readable action steps."""
+        steps: list[str] = []
+
+        source_label = source_obj["label"] if source_obj else source_query
+        dest_label = dest_obj["label"] if dest_obj else dest_query
+
+        # Step 1: Locate source
+        if source_obj:
+            bbox = source_obj["bbox"]
+            steps.append(
+                f"Step 1: Locate '{source_label}' at position "
+                f"[{bbox[0]}, {bbox[1]}, {bbox[2]}, {bbox[3]}]"
+            )
+        else:
+            steps.append(
+                f"Step 1: Locate '{source_label}' — ⚠ Not detected in scene"
+            )
+
+        # Step 2: Locate destination (if applicable)
+        if dest_query:
+            if dest_obj:
+                bbox = dest_obj["bbox"]
+                steps.append(
+                    f"Step 2: Locate destination '{dest_label}' at position "
+                    f"[{bbox[0]}, {bbox[1]}, {bbox[2]}, {bbox[3]}]"
+                )
+            else:
+                steps.append(
+                    f"Step 2: Locate destination '{dest_label}' "
+                    f"— ⚠ Not detected in scene"
+                )
+
+        # Step 3: Execute action
+        action_desc = action.replace("_", " ").capitalize()
+        if dest_label:
+            steps.append(
+                f"Step {len(steps) + 1}: {action_desc} '{source_label}' "
+                f"→ '{dest_label}'"
+            )
+        else:
+            steps.append(
+                f"Step {len(steps) + 1}: {action_desc} '{source_label}'"
+            )
+
+        # Step 4: Confirmation
+        steps.append(f"Step {len(steps) + 1}: Verify action completion")
+
+        return steps
+
+    @staticmethod
+    def _determine_status(
+        source_obj: dict | None,
+        dest_obj: dict | None,
+        dest_query: str | None,
+    ) -> str:
+        """Determine planning status based on matching results."""
+        if source_obj is None:
+            return "failed"
+        if dest_query and dest_obj is None:
+            return "partial"
+        return "success"
+
+    @staticmethod
+    def _empty_plan(reason: str) -> dict[str, Any]:
+        """Return an empty plan with a failure reason."""
+        return {
+            "action": "none",
+            "steps": [f"Planning failed: {reason}"],
+            "source_object": None,
+            "dest_object": None,
+            "source_bbox": None,
+            "dest_bbox": None,
+            "status": "failed",
+        }

src/ui/__init__.py

@@ -0,0 +1,12 @@
+"""
+src/ui — Gradio UI package for the VLA Demo System.
+
+Exports
+-------
+create_ui : callable
+    Factory function that builds and returns the Gradio Blocks interface.
+"""
+
+from src.ui.layout import create_ui
+
+__all__ = ["create_ui"]

src/ui/components.py

@@ -0,0 +1,99 @@
+"""
+src/ui/components.py — Reusable HTML components for the Codevally VLA UI.
+
+Each constant is a self-contained HTML string ready to drop into a
+``gr.HTML()`` call. Keeping HTML here avoids polluting layout.py with
+long inline strings.
+"""
+
+HEADER_HTML: str = """
+<div class="cv-header">
+  <div style="display:flex; align-items:center; gap:16px; margin-bottom:14px;
+              position:relative; z-index:1;">
+    <div style="width:44px; height:44px;
+                background:linear-gradient(135deg,#1A4BDB,#00D4FF);
+                border-radius:12px; display:flex; align-items:center;
+                justify-content:center; font-size:22px; flex-shrink:0;
+                box-shadow:0 4px 16px rgba(0,212,255,0.3);">🤖</div>
+    <div>
+      <div style="font-size:22px; font-weight:700; color:#fff;
+                  letter-spacing:-0.01em; line-height:1.2;">
+        Vision Language Action
+      </div>
+      <div style="font-size:13px; color:rgba(255,255,255,0.5); margin-top:2px;">
+        Powered by
+        <a href="https://www.codevally.com" target="_blank"
+           style="color:#00D4FF; font-weight:600; text-decoration:none;">
+          Codevally
+        </a>
+        &nbsp;·&nbsp; Open Vocabulary Detection + LLM Reasoning
+      </div>
+    </div>
+    <div style="margin-left:auto; display:flex; flex-direction:column;
+                align-items:flex-end; gap:6px; position:relative; z-index:1;">
+      <a href="https://www.codevally.com" target="_blank"
+         style="background:rgba(0,212,255,0.12); border:1px solid rgba(0,212,255,0.35);
+                border-radius:8px; padding:7px 18px; color:#00D4FF; font-size:13px;
+                font-weight:600; text-decoration:none; letter-spacing:0.02em;
+                transition:background 0.2s;"
+         onmouseover="this.style.background='rgba(0,212,255,0.2)'"
+         onmouseout="this.style.background='rgba(0,212,255,0.12)'">
+        🌐 codevally.com
+      </a>
+    </div>
+  </div>
+
+  <p style="font-size:14px; color:rgba(255,255,255,0.6); max-width:760px;
+            line-height:1.7; margin:0 0 14px; position:relative; z-index:1;">
+    An enterprise-grade
+    <strong style="color:rgba(255,255,255,0.85);">
+      Vision Language Action (VLA) pipeline
+    </strong>
+    that understands any natural-language instruction, detects objects using
+    open vocabulary AI, generates structured action plans, and visualizes
+    robotic task execution — all in real time.
+  </p>
+
+  <div style="display:flex; flex-wrap:wrap; gap:6px;
+              position:relative; z-index:1; margin-bottom:0;">
+    <span class="cv-badge">🔍 Grounding DINO</span>
+    <span class="cv-badge">🧠 GPT-4o</span>
+    <span class="cv-badge">🎯 Open Vocabulary</span>
+    <span class="cv-badge">⚡ Real-time Inference</span>
+    <span class="cv-badge">🤖 Action Planning</span>
+    <span class="cv-badge">🏭 Industrial AI</span>
+  </div>
+
+  <div class="how-it-works">
+    <div class="hiw-step">
+      <span class="hiw-icon">📸</span>
+      <span class="hiw-label">Vision</span>
+      Open vocabulary object detection
+    </div>
+    <div class="hiw-step">
+      <span class="hiw-icon">🧠</span>
+      <span class="hiw-label">Language</span>
+      LLM command interpretation
+    </div>
+    <div class="hiw-step">
+      <span class="hiw-icon">📋</span>
+      <span class="hiw-label">Planning</span>
+      Structured action steps
+    </div>
+    <div class="hiw-step">
+      <span class="hiw-icon">🎬</span>
+      <span class="hiw-label">Action</span>
+      Annotated visualization
+    </div>
+  </div>
+</div>
+"""
+
+FOOTER_HTML: str = """
+<div class="cv-footer">
+  Built by
+  <a href="https://www.codevally.com" target="_blank">Codevally</a>
+  &nbsp;·&nbsp; Vision Language Action AI Platform
+  &nbsp;·&nbsp; © 2026 Codevally Technologies
+</div>
+"""

src/ui/examples.py

@@ -0,0 +1,45 @@
+"""
+src/ui/examples.py — Demo image loader for the Gradio Examples component.
+
+Reads demo images from the configured assets directory and pairs each
+image with a contextually appropriate natural-language command.
+"""
+
+from pathlib import Path
+
+from src.utils.config import DEMO_IMAGES_DIR
+
+
+# Map each demo image filename to a curated command
+_IMAGE_COMMANDS: dict[str, str] = {
+    "assembly_line.jpg":     "Pick up the defective board and place it in the red bin",
+    "construction_site.jpg": "Move the hard hat next to the gloves",
+    "desk_scene.jpg":        "Pick up the bottle and place it in the box",
+    "garage_tools.jpg":      "Place the wrench back into the toolbox",
+    "grocery_shelf.jpg":     "Move the water bottle into the shopping basket",
+    "kitchen_counter.jpg":   "Pick up the apple and place it in the bowl",
+    "lab_equipment.jpg":     "Move the bottle next to the microscope",
+    "medical_tray.jpg":      "Pick up the scissors and place them on the tray",
+    "office_supplies.jpg":   "Move the phone next to the laptop",
+    "warehouse_boxes.jpg":   "Move the closest box onto the hand truck",
+}
+
+
+def get_demo_examples() -> list[list]:
+    """
+    Return a list of ``[image_path, command]`` pairs for Gradio Examples.
+
+    Returns:
+        Sorted list of ``[str, str]`` pairs, one per demo image found on disk.
+        Returns an empty list if the demo images directory does not exist.
+    """
+    demo_dir = Path(DEMO_IMAGES_DIR)
+    if not demo_dir.exists():
+        return []
+
+    examples: list[list] = []
+    for img_path in sorted(demo_dir.glob("*.jpg")):
+        cmd = _IMAGE_COMMANDS.get(img_path.name, "Inspect the closest object")
+        examples.append([str(img_path), cmd])
+
+    return examples

src/ui/handlers.py

@@ -0,0 +1,146 @@
+"""
+src/ui/handlers.py — Gradio event-handler functions for the VLA pipeline.
+
+These functions are wired to Gradio UI events (button clicks, etc.) and
+are intentionally kept free of Gradio component construction so they can
+be unit-tested independently.
+"""
+
+import json
+import logging
+
+import cv2
+import numpy as np
+
+from src.pipeline.vla_pipeline import VLAPipeline
+
+logger = logging.getLogger(__name__)
+
+# ---------------------------------------------------------------------------
+# Shared pipeline instance (initialised once at import time)
+# ---------------------------------------------------------------------------
+_pipeline: VLAPipeline | None = None
+
+
+def get_pipeline() -> VLAPipeline:
+    """Return the shared VLAPipeline, initialising it on first call."""
+    global _pipeline
+    if _pipeline is None:
+        logger.info("Initializing VLA Pipeline...")
+        _pipeline = VLAPipeline()
+        logger.info("VLA Pipeline ready.")
+    return _pipeline
+
+
+# ---------------------------------------------------------------------------
+# Main Gradio handler
+# ---------------------------------------------------------------------------
+def process_pipeline(
+    image: np.ndarray | None,
+    command: str,
+) -> tuple:
+    """
+    Run the VLA pipeline and return outputs for Gradio components.
+
+    Args:
+        image:   Input image as a NumPy array (from gr.Image).
+        command: Natural-language instruction from the user.
+
+    Returns:
+        5-tuple:
+            - detection_image (np.ndarray)
+            - detections_json (str)
+            - action_plan_md  (str)
+            - action_image    (np.ndarray)
+            - status_message  (str)
+    """
+    empty = _blank_image("Upload an image and\nenter a command to begin")
+
+    if image is None:
+        return empty, "[]", "*Please upload an image.*", empty, "⚠️ No image provided."
+
+    if not command or not command.strip():
+        return empty, "[]", "*Please enter a command.*", empty, "⚠️ No command entered."
+
+    logger.info("Processing: '%s'", command)
+    result = get_pipeline().run(image, command)
+
+    detections_json = json.dumps(result.detections, indent=2) if result.detections else "[]"
+    action_plan_md = _format_action_plan(result.action_plan, result.interpretation)
+
+    # Status bar
+    if result.error:
+        status = f"⚠️ {result.error}"
+    else:
+        plan_status = result.action_plan.get("status", "unknown")
+        emoji = {"success": "✅", "partial": "⚠️", "failed": "❌"}.get(plan_status, "❓")
+        n = len(result.detections)
+        steps = len(result.action_plan.get("steps", []))
+        status = (
+            f"{emoji} Pipeline complete — Plan status: **{plan_status}**"
+            f" |  {n} object{'s' if n != 1 else ''} detected"
+            f" |  {steps} action step{'s' if steps != 1 else ''} generated"
+        )
+
+    det_img = _ensure_rgb(result.detection_image, empty)
+    act_img = _ensure_rgb(result.action_image, empty)
+    return det_img, detections_json, action_plan_md, act_img, status
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+def _format_action_plan(action_plan: dict, interpretation: dict) -> str:
+    """Format the action plan dict as a readable Markdown string."""
+    if not action_plan:
+        return "*No action plan generated.*"
+
+    lines: list[str] = []
+
+    if interpretation:
+        lines += [
+            "### 🧠 Command Interpretation",
+            "| Field | Value |",
+            "|-------|-------|",
+            f"| **Action** | `{interpretation.get('action', 'N/A')}` |",
+            f"| **Source** | `{interpretation.get('source', 'N/A')}` |",
+            f"| **Destination** | `{interpretation.get('destination') or 'N/A'}` |",
+            "",
+        ]
+
+    steps = action_plan.get("steps", [])
+    if steps:
+        lines.append("### 📋 Execution Steps")
+        for i, step in enumerate(steps, 1):
+            lines.append(f"**{i}.** {step}")
+        lines.append("")
+
+    status = action_plan.get("status", "unknown")
+    emoji = {"success": "✅", "partial": "⚠️", "failed": "❌"}.get(status, "❓")
+    lines += ["---", f"### {emoji} Status: `{status.upper()}`"]
+
+    return "\n".join(lines)
+
+
+def _ensure_rgb(image: np.ndarray | None, fallback: np.ndarray) -> np.ndarray:
+    """Convert image to RGB format expected by Gradio."""
+    if image is None:
+        return fallback
+    if len(image.shape) == 2:
+        return cv2.cvtColor(image, cv2.COLOR_GRAY2RGB)
+    if image.shape[2] == 4:
+        return cv2.cvtColor(image, cv2.COLOR_BGRA2RGB)
+    return image
+
+
+def _blank_image(text: str = "") -> np.ndarray:
+    """Return a dark placeholder image with optional centred text."""
+    img = np.full((400, 600, 3), 18, dtype=np.uint8)  # near-black
+    if text:
+        for i, line in enumerate(text.split("\n")):
+            y = 185 + i * 34
+            (w, _), _ = cv2.getTextSize(line, cv2.FONT_HERSHEY_SIMPLEX, 0.7, 1)
+            x = max(0, (600 - w) // 2)
+            cv2.putText(img, line, (x, y), cv2.FONT_HERSHEY_SIMPLEX,
+                        0.7, (100, 100, 100), 1, cv2.LINE_AA)
+    return img

src/ui/layout.py

@@ -0,0 +1,151 @@
+"""
+src/ui/layout.py — Gradio Blocks layout definition for the VLA Demo.
+
+This module owns *only* the component hierarchy and wiring.
+Styling   → src/ui/styles.py
+HTML bits → src/ui/components.py
+Handlers  → src/ui/handlers.py
+Examples  → src/ui/examples.py
+"""
+
+import gradio as gr
+
+from src.ui.styles import CUSTOM_CSS
+from src.ui.components import HEADER_HTML, FOOTER_HTML
+from src.ui.handlers import process_pipeline
+from src.ui.examples import get_demo_examples
+
+
+def create_ui() -> gr.Blocks:
+    """
+    Build and return the Gradio Blocks interface.
+
+    All sub-modules are imported locally so that this function can be
+    called in isolation for testing without triggering heavy model loads.
+    """
+    demo_examples = get_demo_examples()
+
+    with gr.Blocks(
+        title="VLA — AI Vision Language Action Pipeline | Codevally",
+        theme=gr.themes.Base(
+            primary_hue="blue",
+            secondary_hue="cyan",
+            neutral_hue="slate",
+            font=[gr.themes.GoogleFont("Inter"), "system-ui", "sans-serif"],
+        ),
+        css=CUSTOM_CSS,
+    ) as demo:
+
+        # ── Header ────────────────────────────────────────────────────────
+        gr.HTML(HEADER_HTML)
+
+        # ── Main layout ───────────────────────────────────────────────────
+        with gr.Row(equal_height=False):
+
+            # ── Left: Inputs ───────────────────────────────────────────
+            with gr.Column(scale=1, min_width=300):
+                gr.Markdown("<div class='cv-section-label'>📷 Input</div>")
+                image_input = gr.Image(
+                    label="Upload Scene Image",
+                    type="numpy",
+                    height=280,
+                    elem_id="image_upload",
+                )
+                gr.Markdown(
+                    "<div class='cv-section-label' style='margin-top:12px;'>"
+                    "💬 Natural Language Command</div>"
+                )
+                command_input = gr.Textbox(
+                    label="Command",
+                    placeholder=(
+                        'e.g. "Pick up the defective board and place it in the red bin"'
+                    ),
+                    lines=3,
+                    show_label=False,
+                )
+                run_btn = gr.Button(
+                    "🚀 Run AI Pipeline",
+                    variant="primary",
+                    size="lg",
+                )
+                gr.Markdown(
+                    "<div class='cv-section-label' style='margin-top:14px;'>"
+                    "📊 Pipeline Status</div>"
+                )
+                status_output = gr.Markdown(
+                    value="*Ready — upload a scene image and enter an instruction.*",
+                    elem_classes=["status-box"],
+                )
+
+            # ── Right: Outputs ─────────────────────────────────────────
+            with gr.Column(scale=2):
+                with gr.Tabs(elem_id="result_tabs"):
+
+                    with gr.TabItem("🔍 Detection Results"):
+                        gr.Markdown(
+                            "_Grounding DINO detects scene objects based on your "
+                            "command text — no fixed class limitations._"
+                        )
+                        with gr.Row():
+                            detection_image = gr.Image(
+                                label="Detected Objects",
+                                type="numpy",
+                                height=320,
+                            )
+                            detections_json = gr.Code(
+                                label="Detection Data (JSON)",
+                                language="json",
+                                lines=14,
+                            )
+
+                    with gr.TabItem("🎯 Action Plan & Visualization"):
+                        gr.Markdown(
+                            "_The LLM interprets your command and maps it to detected "
+                            "objects, generating step-by-step robotic action instructions._"
+                        )
+                        with gr.Row():
+                            action_image = gr.Image(
+                                label="Action Visualization",
+                                type="numpy",
+                                height=320,
+                            )
+                            action_plan_md = gr.Markdown(
+                                value=(
+                                    "*Run the pipeline to see the "
+                                    "AI-generated action plan.*"
+                                ),
+                            )
+
+        # ── Demo examples ──────────────────────────────────────────────────
+        if demo_examples:
+            gr.Markdown("---")
+            gr.Markdown(
+                "### 📂 Try a Demo Scene\n"
+                "_Click any example below to load a scene image with a pre-filled "
+                "command. These showcase Codevally's VLA pipeline across diverse "
+                "industrial and everyday environments._"
+            )
+            gr.Examples(
+                examples=demo_examples,
+                inputs=[image_input, command_input],
+                label="Demo Scenes",
+                examples_per_page=5,
+            )
+
+        # ── Footer ────────────────────────────────────────────────────────
+        gr.HTML(FOOTER_HTML)
+
+        # ── Event wiring ──────────────────────────────────────────────────
+        run_btn.click(
+            fn=process_pipeline,
+            inputs=[image_input, command_input],
+            outputs=[
+                detection_image,
+                detections_json,
+                action_plan_md,
+                action_image,
+                status_output,
+            ],
+        )
+
+    return demo

src/ui/styles.py

@@ -0,0 +1,192 @@
+"""
+src/ui/styles.py — Custom CSS for the Codevally VLA Demo UI.
+
+All visual styling is centralised here so that layout.py stays
+focused purely on component structure.
+"""
+
+CUSTOM_CSS: str = """
+/* ── Google Font ── */
+@import url('https://fonts.googleapis.com/css2?family=Inter:wght@300;400;500;600;700&display=swap');
+
+/* ── Root & Body ── */
+:root {
+    --cv-blue:    #1A4BDB;
+    --cv-dark:    #0D1B3E;
+    --cv-accent:  #00D4FF;
+    --cv-glass:   rgba(255,255,255,0.06);
+    --cv-border:  rgba(255,255,255,0.1);
+    --cv-success: #22C55E;
+    --cv-warn:    #F59E0B;
+    --cv-error:   #EF4444;
+}
+
+body, .gradio-container {
+    font-family: 'Inter', system-ui, sans-serif !important;
+    background: linear-gradient(135deg, #0a0f1e 0%, #0d1b3e 50%, #0a0f1e 100%) !important;
+    min-height: 100vh;
+}
+
+/* ── Header Banner ── */
+.cv-header {
+    background: linear-gradient(135deg, #0D1B3E 0%, #1A2E5E 60%, #0D1B3E 100%);
+    border: 1px solid var(--cv-border);
+    border-radius: 16px;
+    padding: 28px 36px;
+    margin-bottom: 20px;
+    position: relative;
+    overflow: hidden;
+}
+.cv-header::before {
+    content: '';
+    position: absolute;
+    top: -60px; right: -60px;
+    width: 220px; height: 220px;
+    background: radial-gradient(circle, rgba(0,212,255,0.12) 0%, transparent 70%);
+    border-radius: 50%;
+}
+.cv-header::after {
+    content: '';
+    position: absolute;
+    bottom: -40px; left: 20%;
+    width: 140px; height: 140px;
+    background: radial-gradient(circle, rgba(26,75,219,0.15) 0%, transparent 70%);
+    border-radius: 50%;
+}
+
+/* ── Badges ── */
+.cv-badge {
+    display: inline-block;
+    background: rgba(0,212,255,0.12);
+    color: var(--cv-accent);
+    border: 1px solid rgba(0,212,255,0.3);
+    border-radius: 20px;
+    padding: 3px 12px;
+    font-size: 12px;
+    font-weight: 500;
+    margin: 2px 4px 2px 0;
+    letter-spacing: 0.03em;
+}
+
+/* ── Cards ── */
+.cv-card {
+    background: var(--cv-glass);
+    backdrop-filter: blur(12px);
+    border: 1px solid var(--cv-border);
+    border-radius: 16px;
+    padding: 20px;
+    transition: border-color 0.2s;
+}
+.cv-card:hover { border-color: rgba(0,212,255,0.25); }
+
+/* ── Section labels ── */
+.cv-section-label {
+    font-size: 11px;
+    font-weight: 600;
+    letter-spacing: 0.08em;
+    text-transform: uppercase;
+    color: var(--cv-accent);
+    margin-bottom: 8px;
+}
+
+/* ── Run button ── */
+button.primary {
+    background: linear-gradient(135deg, var(--cv-blue) 0%, #2563EB 100%) !important;
+    border: none !important;
+    border-radius: 10px !important;
+    font-weight: 600 !important;
+    font-size: 15px !important;
+    letter-spacing: 0.02em !important;
+    box-shadow: 0 4px 20px rgba(26,75,219,0.4) !important;
+    transition: transform 0.15s, box-shadow 0.15s !important;
+}
+button.primary:hover {
+    transform: translateY(-1px) !important;
+    box-shadow: 0 6px 28px rgba(26,75,219,0.55) !important;
+}
+button.primary:active { transform: translateY(0) !important; }
+
+/* ── Tabs ── */
+.tabs .tab-nav { border-bottom: 1px solid var(--cv-border) !important; }
+.tabs .tab-nav button {
+    font-weight: 500 !important;
+    color: rgba(255,255,255,0.55) !important;
+    border-radius: 8px 8px 0 0 !important;
+}
+.tabs .tab-nav button.selected {
+    color: var(--cv-accent) !important;
+    border-bottom: 2px solid var(--cv-accent) !important;
+    background: rgba(0,212,255,0.07) !important;
+}
+
+/* ── Inputs ── */
+input, textarea, .block { border-radius: 10px !important; }
+
+/* ── Footer ── */
+.cv-footer {
+    text-align: center;
+    padding: 16px 0 6px;
+    font-size: 13px;
+    color: rgba(255,255,255,0.35);
+}
+.cv-footer a {
+    color: var(--cv-accent);
+    text-decoration: none;
+    font-weight: 500;
+}
+.cv-footer a:hover { text-decoration: underline; }
+
+/* ── Status box ── */
+.status-box {
+    background: rgba(255,255,255,0.04);
+    border-radius: 10px;
+    border: 1px solid var(--cv-border);
+    padding: 10px 14px;
+    font-size: 14px;
+    min-height: 40px;
+}
+
+/* ── How it works strip ── */
+.how-it-works {
+    display: flex;
+    gap: 0;
+    margin: 18px 0 8px;
+    border-radius: 12px;
+    overflow: hidden;
+    border: 1px solid var(--cv-border);
+}
+.hiw-step {
+    flex: 1;
+    padding: 14px 12px;
+    text-align: center;
+    background: rgba(255,255,255,0.03);
+    border-right: 1px solid var(--cv-border);
+    font-size: 12px;
+    color: rgba(255,255,255,0.7);
+}
+.hiw-step:last-child { border-right: none; }
+.hiw-step .hiw-icon  { font-size: 20px; display: block; margin-bottom: 4px; }
+.hiw-step .hiw-label {
+    font-weight: 600; font-size: 11px; letter-spacing: 0.05em;
+    color: var(--cv-accent); display: block; margin-bottom: 2px;
+    text-transform: uppercase;
+}
+
+/* ── Hide default Gradio footer bar ── */
+footer { display: none !important; }
+.gradio-container ~ footer { display: none !important; }
+
+/* ─�� Scrollable Detection JSON box ── */
+#result_tabs .code-wrap,
+#result_tabs .cm-scroller,
+#result_tabs .codemirror-wrapper {
+    max-height: 380px !important;
+    overflow-y: auto !important;
+    scrollbar-width: thin;
+    scrollbar-color: rgba(0,212,255,0.4) rgba(255,255,255,0.05);
+}
+#result_tabs .code-wrap::-webkit-scrollbar { width: 6px; }
+#result_tabs .code-wrap::-webkit-scrollbar-track { background: rgba(255,255,255,0.05); border-radius: 3px; }
+#result_tabs .code-wrap::-webkit-scrollbar-thumb { background: rgba(0,212,255,0.4); border-radius: 3px; }
+#result_tabs .code-wrap::-webkit-scrollbar-thumb:hover { background: rgba(0,212,255,0.65); }
+"""

src/utils/__init__.py

@@ -0,0 +1 @@
+# Utilities Package

src/utils/config.py

@@ -0,0 +1,100 @@
+"""
+Configuration module for the VLA Demo System.
+
+Loads settings from environment variables (with .env file support).
+All configuration is centralized here to avoid hardcoded values.
+"""
+
+import os
+import logging
+from pathlib import Path
+
+from dotenv import load_dotenv
+
+# Load .env from project root
+_PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent
+load_dotenv(_PROJECT_ROOT / ".env")
+
+logger = logging.getLogger(__name__)
+
+# ---------------------------------------------------------------------------
+# LLM Provider Configuration
+# ---------------------------------------------------------------------------
+# LLM_PROVIDER selects the language model backend: "openai" or "groq"
+LLM_PROVIDER: str = os.getenv("LLM_PROVIDER", "openai")  # "openai" or "groq"
+
+OPENAI_API_KEY: str = os.getenv("OPENAI_API_KEY", "")
+OPENAI_MODEL: str = os.getenv("OPENAI_MODEL", "gpt-4o-mini")
+
+GROQ_API_KEY: str = os.getenv("GROQ_API_KEY", "")
+GROQ_MODEL: str = os.getenv("GROQ_MODEL", "llama-3.3-70b-versatile")
+
+# ---------------------------------------------------------------------------
+# Vision / Detection Configuration
+# ---------------------------------------------------------------------------
+DETECTION_MODEL: str = os.getenv("DETECTION_MODEL", "grounding_dino")  # "yolo" or "grounding_dino"
+YOLO_MODEL: str = os.getenv("YOLO_MODEL", "yolov8n.pt")
+YOLO_CONFIDENCE: float = float(os.getenv("YOLO_CONFIDENCE", "0.25"))
+GROUNDING_DINO_MODEL: str = os.getenv(
+    "GROUNDING_DINO_MODEL", "IDEA-Research/grounding-dino-tiny"
+)
+GROUNDING_DINO_CONFIDENCE: float = float(os.getenv("GROUNDING_DINO_CONFIDENCE", "0.3"))
+
+# ---------------------------------------------------------------------------
+# Paths
+# ---------------------------------------------------------------------------
+ASSETS_DIR: Path = _PROJECT_ROOT / "assets"
+DEMO_IMAGES_DIR: Path = ASSETS_DIR / "demo_images"
+MODELS_DIR: Path = _PROJECT_ROOT / "models"
+
+# ---------------------------------------------------------------------------
+# Logging Setup
+# ---------------------------------------------------------------------------
+LOG_LEVEL: str = os.getenv("LOG_LEVEL", "INFO")
+
+
+def setup_logging() -> None:
+    """Configure root logger with a consistent format."""
+    logging.basicConfig(
+        level=getattr(logging, LOG_LEVEL.upper(), logging.INFO),
+        format="%(asctime)s | %(levelname)-8s | %(name)s | %(message)s",
+        datefmt="%Y-%m-%d %H:%M:%S",
+    )
+
+
+def validate_config() -> list[str]:
+    """
+    Validate critical configuration values.
+
+    Returns:
+        List of warning messages for missing/invalid config values.
+    """
+    warnings: list[str] = []
+
+    valid_providers = ("openai", "groq")
+    if LLM_PROVIDER not in valid_providers:
+        warnings.append(
+            f"Unknown LLM_PROVIDER='{LLM_PROVIDER}'. "
+            f"Valid values: {valid_providers}. Defaulting to 'openai'."
+        )
+
+    if LLM_PROVIDER == "openai" and not OPENAI_API_KEY:
+        warnings.append(
+            "LLM_PROVIDER='openai' but OPENAI_API_KEY is not set. "
+            "Language reasoning will use fallback mode."
+        )
+
+    if LLM_PROVIDER == "groq" and not GROQ_API_KEY:
+        warnings.append(
+            "LLM_PROVIDER='groq' but GROQ_API_KEY is not set. "
+            "Language reasoning will use fallback mode."
+        )
+
+    valid_detectors = ("yolo", "grounding_dino")
+    if DETECTION_MODEL not in valid_detectors:
+        warnings.append(
+            f"Unknown DETECTION_MODEL='{DETECTION_MODEL}'. "
+            f"Valid values: {valid_detectors}. Defaulting to 'grounding_dino'."
+        )
+
+    return warnings

src/utils/object_mapper.py

@@ -0,0 +1,366 @@
+"""
+Object Mapper — Synonym normalization for bridging YOLO labels
+and natural-language user commands.
+
+YOLO may detect "cup" while the user says "mug". This module provides
+a mapping layer and fuzzy-match helper to reconcile the two vocabularies.
+"""
+
+import logging
+from difflib import SequenceMatcher
+
+logger = logging.getLogger(__name__)
+
+# ---------------------------------------------------------------------------
+# Synonym Map: user-language → canonical YOLO label
+# ---------------------------------------------------------------------------
+SYNONYM_MAP: dict[str, str] = {
+    # ---------------------------------------------------------------
+    # COCO has 80 classes. This map bridges common user vocabulary
+    # to the actual YOLO/COCO label strings.
+    # ---------------------------------------------------------------
+
+    # person
+    "human": "person",
+    "worker": "person",
+    "operator": "person",
+    "man": "person",
+    "woman": "person",
+    "people": "person",
+    "employee": "person",
+    "technician": "person",
+
+    # bicycle
+    "bike": "bicycle",
+    "cycle": "bicycle",
+
+    # car
+    "automobile": "car",
+    "vehicle": "car",
+    "sedan": "car",
+
+    # motorcycle
+    "motorbike": "motorcycle",
+    "scooter": "motorcycle",
+
+    # truck
+    "van": "truck",
+    "lorry": "truck",
+    "pickup": "truck",
+
+    # boat
+    "ship": "boat",
+    "vessel": "boat",
+    "canoe": "boat",
+
+    # traffic light
+    "signal": "traffic light",
+    "stoplight": "traffic light",
+
+    # backpack
+    "bag": "backpack",
+    "rucksack": "backpack",
+    "knapsack": "backpack",
+
+    # umbrella
+    "parasol": "umbrella",
+
+    # handbag
+    "purse": "handbag",
+    "clutch": "handbag",
+
+    # suitcase (toolboxes, cases look like suitcases to YOLO)
+    "briefcase": "suitcase",
+    "toolbox": "suitcase",
+    "case": "suitcase",
+    "luggage": "suitcase",
+    "tool box": "suitcase",
+
+    # bottle
+    "water bottle": "bottle",
+    "flask": "bottle",
+    "jug": "bottle",
+    "canteen": "bottle",
+    "container": "bottle",
+
+    # wine glass
+    "goblet": "wine glass",
+    "chalice": "wine glass",
+    "glass": "wine glass",
+
+    # cup
+    "mug": "cup",
+    "teacup": "cup",
+    "coffee cup": "cup",
+    "tumbler": "cup",
+
+    # fork
+    "prong": "fork",
+
+    # knife (wrenches, tools often detected as knife-like)
+    "blade": "knife",
+    "wrench": "knife",
+    "spanner": "knife",
+    "screwdriver": "knife",
+    "tool": "knife",
+
+    # spoon
+    "ladle": "spoon",
+    "scoop": "spoon",
+
+    # bowl
+    "tray": "bowl",
+    "dish": "bowl",
+    "plate": "bowl",
+    "bin": "bowl",
+    "basket": "bowl",
+
+    # banana / apple / orange (fruits)
+    "fruit": "apple",
+
+    # sandwich
+    "sub": "sandwich",
+    "burger": "sandwich",
+    "wrap": "sandwich",
+
+    # pizza
+    "pie": "pizza",
+
+    # cake
+    "pastry": "cake",
+    "dessert": "cake",
+
+    # chair
+    "seat": "chair",
+    "stool": "chair",
+    "office chair": "chair",
+
+    # couch
+    "sofa": "couch",
+    "loveseat": "couch",
+    "settee": "couch",
+
+    # bed
+    "mattress": "bed",
+    "cot": "bed",
+
+    # dining table
+    "desk": "dining table",
+    "table": "dining table",
+    "workbench": "dining table",
+    "bench": "dining table",
+    "counter": "dining table",
+
+    # tv
+    "monitor": "tv",
+    "screen": "tv",
+    "display": "tv",
+    "television": "tv",
+
+    # laptop
+    "notebook": "laptop",
+    "computer": "laptop",
+    "macbook": "laptop",
+
+    # mouse
+    "trackpad": "mouse",
+
+    # remote
+    "controller": "remote",
+    "remote control": "remote",
+
+    # keyboard
+    "keypad": "keyboard",
+
+    # cell phone
+    "mobile": "cell phone",
+    "phone": "cell phone",
+    "smartphone": "cell phone",
+    "iphone": "cell phone",
+    "handset": "cell phone",
+    "scanner": "cell phone",
+
+    # microwave
+    "microwave oven": "microwave",
+
+    # oven
+    "stove": "oven",
+    "furnace": "oven",
+
+    # refrigerator
+    "fridge": "refrigerator",
+    "cooler": "refrigerator",
+    "freezer": "refrigerator",
+
+    # book
+    "notebook paper": "book",
+    "manual": "book",
+    "textbook": "book",
+    "binder": "book",
+
+    # clock
+    "watch": "clock",
+    "timer": "clock",
+
+    # vase
+    "pot": "vase",
+    "planter": "vase",
+    "jar": "vase",
+
+    # scissors
+    "cutter": "scissors",
+    "shears": "scissors",
+    "clipper": "scissors",
+    "pliers": "scissors",
+    "wire cutter": "scissors",
+
+    # teddy bear
+    "stuffed animal": "teddy bear",
+    "plush": "teddy bear",
+    "toy": "teddy bear",
+
+    # potted plant
+    "plant": "potted plant",
+    "houseplant": "potted plant",
+    "flower": "potted plant",
+
+    # -----------------------------------------------------------
+    # Domain-specific aliases for industrial / demo contexts
+    # -----------------------------------------------------------
+    "damaged part": "knife",
+    "defective part": "knife",
+    "component": "knife",
+    "metal part": "knife",
+    "part": "knife",
+    "piece": "knife",
+
+    "red bin": "bowl",
+    "blue bin": "bowl",
+    "sorting bin": "bowl",
+    "reject bin": "bowl",
+    "accept bin": "bowl",
+
+    "circuit board": "keyboard",
+    "pcb": "keyboard",
+    "board": "keyboard",
+
+    "hard hat": "sports ball",
+    "helmet": "sports ball",
+    "safety hat": "sports ball",
+
+    "gloves": "backpack",
+    "safety gloves": "backpack",
+
+    "pallet": "suitcase",
+    "crate": "suitcase",
+    "carton": "suitcase",
+    "box": "suitcase",
+
+    "hand truck": "bicycle",
+    "dolly": "bicycle",
+    "cart": "bicycle",
+
+    "forklift": "truck",
+
+    "clipboard": "book",
+    "blueprint": "book",
+
+    "stethoscope": "tie",
+    "syringe": "toothbrush",
+    "safety goggles": "wine glass",
+    "goggles": "wine glass",
+
+    "oil can": "bottle",
+    "spray can": "bottle",
+    "battery": "suitcase",
+    "car battery": "suitcase",
+
+    "multimeter": "cell phone",
+    "meter": "cell phone",
+    "soldering iron": "knife",
+    "magnifying lamp": "clock",
+}
+
+
+
+def normalize_label(label: str) -> str:
+    """
+    Normalize a label using the synonym map.
+
+    Args:
+        label: Raw label string (from user command or YOLO).
+
+    Returns:
+        The canonical label if a synonym exists, otherwise the
+        original label lowercased and stripped.
+    """
+    cleaned = label.strip().lower()
+    return SYNONYM_MAP.get(cleaned, cleaned)
+
+
+def _similarity(a: str, b: str) -> float:
+    """Return a 0-1 similarity score between two strings."""
+    return SequenceMatcher(None, a.lower(), b.lower()).ratio()
+
+
+def find_best_match(
+    query: str,
+    detected_labels: list[str],
+    threshold: float = 0.4,
+) -> str | None:
+    """
+    Find the best matching detected label for a user query term.
+
+    Strategy:
+        1. Exact match after normalization.
+        2. Substring containment check.
+        3. Fuzzy similarity above *threshold*.
+
+    Args:
+        query: The term from the user command (e.g. "mug").
+        detected_labels: Labels returned by the vision detector.
+        threshold: Minimum similarity score for fuzzy matching.
+
+    Returns:
+        The best matching label, or ``None`` if no match is found.
+    """
+    normalized_query = normalize_label(query)
+    normalized_detected = {normalize_label(lbl): lbl for lbl in detected_labels}
+
+    # 1. Exact match on normalized labels
+    if normalized_query in normalized_detected:
+        logger.info(
+            "Exact match: '%s' → '%s'",
+            query,
+            normalized_detected[normalized_query],
+        )
+        return normalized_detected[normalized_query]
+
+    # 2. Substring containment
+    for norm_lbl, orig_lbl in normalized_detected.items():
+        if normalized_query in norm_lbl or norm_lbl in normalized_query:
+            logger.info(
+                "Substring match: '%s' → '%s'", query, orig_lbl
+            )
+            return orig_lbl
+
+    # 3. Fuzzy matching
+    best_score = 0.0
+    best_label: str | None = None
+    for norm_lbl, orig_lbl in normalized_detected.items():
+        score = _similarity(normalized_query, norm_lbl)
+        if score > best_score:
+            best_score = score
+            best_label = orig_lbl
+
+    if best_label and best_score >= threshold:
+        logger.info(
+            "Fuzzy match: '%s' → '%s' (score=%.2f)",
+            query,
+            best_label,
+            best_score,
+        )
+        return best_label
+
+    logger.warning("No match found for query '%s'", query)
+    return None

src/vision/__init__.py

@@ -0,0 +1 @@
+# Vision Module Package

src/vision/detector.py

@@ -0,0 +1,134 @@
+"""
+Vision Module — Object detection using YOLOv8.
+
+Wraps the Ultralytics YOLO model to detect objects in images and return
+structured results. The model is loaded **once** in the constructor and
+reused for all subsequent detections.
+"""
+
+import logging
+from typing import Any
+
+import numpy as np
+
+logger = logging.getLogger(__name__)
+
+
+class ObjectDetector:
+    """
+    Detects objects in images using a pretrained YOLOv8 model.
+
+    Attributes:
+        model: The loaded YOLO model instance (cached after first load).
+    """
+
+    def __init__(self, model_name: str = "yolov8n.pt", confidence: float = 0.25):
+        """
+        Initialize the detector and load the YOLO model.
+
+        Args:
+            model_name: Name or path of the YOLO model weights.
+            confidence: Minimum confidence threshold for detections.
+        """
+        self.model_name = model_name
+        self.confidence = confidence
+        self.model = None
+
+        self._load_model()
+
+    def _load_model(self) -> None:
+        """Load the YOLO model. Called once during initialization."""
+        try:
+            from ultralytics import YOLO
+
+            logger.info("Loading YOLO model: %s", self.model_name)
+            self.model = YOLO(self.model_name)
+            logger.info("YOLO model loaded successfully.")
+        except Exception as exc:
+            logger.error("Failed to load YOLO model '%s': %s", self.model_name, exc)
+            raise RuntimeError(
+                f"Could not load YOLO model '{self.model_name}'. "
+                f"Ensure ultralytics is installed and the model file is accessible."
+            ) from exc
+
+    def detect(self, image: np.ndarray) -> list[dict[str, Any]]:
+        """
+        Run object detection on an image.
+
+        Args:
+            image: Input image as a NumPy array (H, W, C) in BGR or RGB format.
+
+        Returns:
+            List of detected objects, each containing:
+                - ``label`` (str): Class name.
+                - ``confidence`` (float): Detection confidence (0-1).
+                - ``bbox`` (list[int]): Bounding box as [x1, y1, x2, y2].
+
+        Raises:
+            ValueError: If the input image is invalid.
+            RuntimeError: If the model is not loaded.
+        """
+        if self.model is None:
+            raise RuntimeError("YOLO model is not loaded.")
+
+        if image is None or not isinstance(image, np.ndarray):
+            raise ValueError("Invalid image: expected a NumPy ndarray.")
+
+        if image.ndim < 2:
+            raise ValueError(
+                f"Invalid image dimensions: expected 2D or 3D array, "
+                f"got {image.ndim}D."
+            )
+
+        logger.info(
+            "Running detection on image of shape %s with confidence=%.2f",
+            image.shape,
+            self.confidence,
+        )
+
+        try:
+            results = self.model(image, conf=self.confidence, verbose=False)
+        except Exception as exc:
+            logger.error("YOLO inference failed: %s", exc)
+            return []
+
+        detections = self._parse_results(results)
+        logger.info("Detected %d objects.", len(detections))
+        return detections
+
+    def _parse_results(self, results: Any) -> list[dict[str, Any]]:
+        """
+        Parse YOLO results into a structured list of detections.
+
+        Args:
+            results: Raw YOLO results object.
+
+        Returns:
+            Structured list of detection dictionaries.
+        """
+        detections: list[dict[str, Any]] = []
+
+        for result in results:
+            boxes = result.boxes
+            if boxes is None:
+                continue
+
+            for i in range(len(boxes)):
+                try:
+                    bbox = boxes.xyxy[i].cpu().numpy().tolist()
+                    confidence = float(boxes.conf[i].cpu().numpy())
+                    class_id = int(boxes.cls[i].cpu().numpy())
+                    label = result.names.get(class_id, f"class_{class_id}")
+
+                    detections.append(
+                        {
+                            "label": label,
+                            "confidence": round(confidence, 4),
+                            "bbox": [int(coord) for coord in bbox],
+                        }
+                    )
+                except (IndexError, KeyError, AttributeError) as exc:
+                    logger.warning("Failed to parse detection %d: %s", i, exc)
+                    continue
+
+        return detections

src/vision/grounding_dino_detector.py

@@ -0,0 +1,350 @@
+"""
+Grounding DINO Detector — Open-vocabulary object detection using
+text prompts via HuggingFace Transformers.
+
+Unlike YOLO (fixed 80 COCO classes), Grounding DINO can detect *any*
+object described in natural language. This makes it ideal for demos
+where users reference domain-specific objects like "defective board",
+"red bin", or "toolbox".
+"""
+
+import logging
+from typing import Any
+
+import numpy as np
+
+logger = logging.getLogger(__name__)
+
+
+class GroundingDINODetector:
+    """
+    Detects objects in images using text-prompted Grounding DINO.
+
+    The model is loaded once in the constructor and reused. Detection
+    requires a text prompt describing what objects to look for.
+
+    Attributes:
+        model: The loaded Grounding DINO model.
+        processor: The image/text processor.
+    """
+
+    def __init__(
+        self,
+        model_name: str = "IDEA-Research/grounding-dino-tiny",
+        confidence: float = 0.25,
+    ):
+        """
+        Initialize the Grounding DINO detector.
+
+        Args:
+            model_name: HuggingFace model identifier.
+            confidence: Minimum confidence (box_threshold) for detections.
+        """
+        self.model_name = model_name
+        self.confidence = confidence
+        self.model = None
+        self.processor = None
+
+        self._load_model()
+
+    def _load_model(self) -> None:
+        """Load the Grounding DINO model and processor."""
+        try:
+            import torch
+            from transformers import AutoProcessor, AutoModelForZeroShotObjectDetection
+
+            logger.info("Loading Grounding DINO model: %s", self.model_name)
+
+            self.processor = AutoProcessor.from_pretrained(self.model_name)
+            self.model = AutoModelForZeroShotObjectDetection.from_pretrained(
+                self.model_name,
+            )
+
+            # Use GPU if available, otherwise CPU
+            self.device = "cuda" if torch.cuda.is_available() else "cpu"
+            self.model = self.model.to(self.device)
+            self.model.eval()
+
+            logger.info(
+                "Grounding DINO loaded successfully (device=%s).", self.device
+            )
+        except Exception as exc:
+            logger.error(
+                "Failed to load Grounding DINO '%s': %s", self.model_name, exc
+            )
+            raise RuntimeError(
+                f"Could not load Grounding DINO model '{self.model_name}'. "
+                f"Ensure transformers and torch are installed."
+            ) from exc
+
+    def detect(
+        self,
+        image: np.ndarray,
+        text_prompt: str | None = None,
+    ) -> list[dict[str, Any]]:
+        """
+        Run open-vocabulary detection on an image.
+
+        Args:
+            image: Input image as a NumPy array (H, W, C) in RGB format.
+            text_prompt: Dot-separated list of object descriptions to detect.
+                         Example: ``"red bin. defective board. toolbox."``
+                         If None, uses a broad default prompt.
+
+        Returns:
+            List of detected objects with label, confidence, and bbox.
+
+        Raises:
+            ValueError: If the input image is invalid.
+            RuntimeError: If the model is not loaded.
+        """
+        if self.model is None or self.processor is None:
+            raise RuntimeError("Grounding DINO model is not loaded.")
+
+        if image is None or not isinstance(image, np.ndarray):
+            raise ValueError("Invalid image: expected a NumPy ndarray.")
+
+        if image.ndim < 2:
+            raise ValueError(
+                f"Invalid image dimensions: expected 2D or 3D array, "
+                f"got {image.ndim}D."
+            )
+
+        if not text_prompt:
+            text_prompt = "object."
+
+        # Ensure prompt ends with a period (required by Grounding DINO)
+        if not text_prompt.strip().endswith("."):
+            text_prompt = text_prompt.strip() + "."
+
+        logger.info(
+            "Running Grounding DINO detection: prompt='%s', confidence=%.2f",
+            text_prompt,
+            self.confidence,
+        )
+
+        try:
+            import torch
+            from PIL import Image as PILImage
+
+            # Convert numpy to PIL Image
+            pil_image = PILImage.fromarray(image)
+
+            # Process inputs
+            inputs = self.processor(
+                images=pil_image,
+                text=text_prompt,
+                return_tensors="pt",
+            ).to(self.device)
+
+            # Run inference
+            with torch.no_grad():
+                outputs = self.model(**inputs)
+
+            # Post-process results (transformers >=5.0 API)
+            results = self.processor.post_process_grounded_object_detection(
+                outputs,
+                threshold=self.confidence,
+                text_threshold=self.confidence,
+                target_sizes=[pil_image.size[::-1]],  # (height, width)
+            )
+
+            detections = self._parse_results(results)
+            detections = self._apply_nms(detections)
+            logger.info("Grounding DINO detected %d objects (after NMS).", len(detections))
+            return detections
+
+        except Exception as exc:
+            logger.error("Grounding DINO inference failed: %s", exc)
+            return []
+
+    def _parse_results(
+        self, results: list[dict[str, Any]]
+    ) -> list[dict[str, Any]]:
+        """Parse Grounding DINO results into structured detections."""
+        detections: list[dict[str, Any]] = []
+
+        if not results:
+            return detections
+
+        result = results[0]  # Single image
+        boxes = result.get("boxes", [])
+        scores = result.get("scores", [])
+        # transformers >=5.0 returns text_labels (list[str]) directly
+        labels = result.get("text_labels") or result.get("labels", [])
+
+        for i in range(len(boxes)):
+            try:
+                bbox = boxes[i].cpu().numpy().tolist()
+                confidence = float(scores[i].cpu().numpy())
+                raw_label = labels[i] if i < len(labels) else "object"
+                label = str(raw_label).strip()
+
+                detections.append(
+                    {
+                        "label": label,
+                        "confidence": float(round(confidence, 4)),
+                        "bbox": [int(coord) for coord in bbox],
+                    }
+                )
+            except (IndexError, AttributeError) as exc:
+                logger.warning("Failed to parse detection %d: %s", i, exc)
+                continue
+
+        return detections
+
+    @staticmethod
+    def _iou(box_a: list[int], box_b: list[int]) -> float:
+        """
+        Compute Intersection-over-Union (IoU) between two bounding boxes.
+
+        Boxes are in [x1, y1, x2, y2] format.
+        """
+        xa1, ya1, xa2, ya2 = box_a
+        xb1, yb1, xb2, yb2 = box_b
+
+        inter_x1 = max(xa1, xb1)
+        inter_y1 = max(ya1, yb1)
+        inter_x2 = min(xa2, xb2)
+        inter_y2 = min(ya2, yb2)
+
+        inter_w = max(0, inter_x2 - inter_x1)
+        inter_h = max(0, inter_y2 - inter_y1)
+        inter_area = inter_w * inter_h
+
+        area_a = max(0, xa2 - xa1) * max(0, ya2 - ya1)
+        area_b = max(0, xb2 - xb1) * max(0, yb2 - yb1)
+        union_area = area_a + area_b - inter_area
+
+        return inter_area / union_area if union_area > 0 else 0.0
+
+    def _apply_nms(
+        self,
+        detections: list[dict[str, Any]],
+        iou_threshold: float = 0.5,
+    ) -> list[dict[str, Any]]:
+        """
+        Apply Non-Max Suppression to remove redundant detections.
+
+        When two boxes overlap significantly (IoU >= iou_threshold), keep the
+        one with the longer/more-specific label (e.g. "red bin" beats "red")
+        or the higher confidence if labels have the same length.
+
+        Args:
+            detections: Raw list of detection dicts.
+            iou_threshold: IoU overlap threshold above which one box is suppressed.
+
+        Returns:
+            Filtered list with duplicates removed.
+        """
+        if len(detections) <= 1:
+            return detections
+
+        # Sort by confidence descending
+        detections = sorted(detections, key=lambda d: d["confidence"], reverse=True)
+        kept: list[dict[str, Any]] = []
+
+        for det in detections:
+            suppress = False
+            for kept_det in kept:
+                iou = self._iou(det["bbox"], kept_det["bbox"])
+                if iou >= iou_threshold:
+                    # Prefer more specific (longer) label; mark current as suppressed
+                    if len(det["label"]) > len(kept_det["label"]):
+                        # Current is more specific — replace the kept one
+                        kept.remove(kept_det)
+                        break  # will be added below
+                    else:
+                        suppress = True
+                        break
+
+            if not suppress:
+                kept.append(det)
+
+        if len(kept) < len(detections):
+            logger.info(
+                "NMS removed %d redundant detections.", len(detections) - len(kept)
+            )
+
+        return kept
+
+    @staticmethod
+    def build_prompt_from_command(command: str) -> str:
+        """
+        Extract object-like noun phrases from a user command and build
+        a Grounding DINO text prompt.
+
+        Condition adjectives (defective, damaged, broken, etc.) are stripped
+        because they describe *state*, not visual appearance, and confuse
+        Grounding DINO. The underlying object noun is kept.
+
+        Args:
+            command: User's natural-language command.
+
+        Returns:
+            A dot-separated prompt string for Grounding DINO.
+
+        Examples:
+            >>> build_prompt_from_command("Move the red bin to the table")
+            "red bin. table."
+            >>> build_prompt_from_command("Pick up the defective board")
+            "board."
+        """
+        import re
+        cmd_lower = command.lower().strip()
+
+        # ------------------------------------------------------------------
+        # Step 1: Strip condition/state adjectives — they describe condition
+        # not visual appearance so Grounding DINO can't detect them.
+        # ------------------------------------------------------------------
+        CONDITION_ADJECTIVES = {
+            "defective", "damaged", "broken", "faulty", "worn", "old",
+            "cracked", "scratched", "dented", "bent", "burnt", "corroded",
+            "rusted", "leaking", "contaminated", "expired", "rejected",
+            "malfunctioning", "misaligned", "missing", "incorrect",
+            "closest", "nearest", "specific", "certain", "particular",
+        }
+        words = cmd_lower.split()
+        cleaned_words = [w for w in words if w not in CONDITION_ADJECTIVES]
+        cmd_lower = " ".join(cleaned_words)
+
+        # ------------------------------------------------------------------
+        # Step 2: Remove action verbs, articles, and prepositions
+        # ------------------------------------------------------------------
+        stop_words = {
+            "the", "a", "an", "and", "or", "to", "from", "in", "on", "at",
+            "into", "onto", "of", "with", "for", "it", "up", "back", "next",
+            "pick", "place", "move", "inspect", "rotate", "remove", "find",
+            "put", "take", "grab", "get", "bring", "carry", "push", "pull",
+            "near", "close", "under", "over", "above", "below", "beside",
+            "behind", "please", "can", "you", "will", "should", "must",
+            "then", "also", "there", "here", "this", "that",
+        }
+
+        # ------------------------------------------------------------------
+        # Step 3: Split on locational prepositions to isolate noun chunks
+        # ------------------------------------------------------------------
+        chunks = re.split(
+            r'\b(?:to|from|into|onto|next to|near|beside|under|over|on top of'
+            r'|and\s+(?:place|put|move|bring|carry|push|pull|drop|set|lay))\b',
+            cmd_lower,
+        )
+
+        object_terms: list[str] = []
+        seen: set[str] = set()
+        for chunk in chunks:
+            chunk_words = chunk.strip().split()
+            cleaned = [w for w in chunk_words if w not in stop_words]
+            if cleaned:
+                term = " ".join(cleaned).strip()
+                # Deduplicate and skip trivially short terms
+                if term and len(term) > 1 and term not in seen:
+                    object_terms.append(term)
+                    seen.add(term)
+
+        if not object_terms:
+            return cmd_lower + "."
+
+        prompt = ". ".join(object_terms) + "."
+        logger.info("Built Grounding DINO prompt: '%s'", prompt)
+        return prompt

src/visualization/__init__.py

@@ -0,0 +1 @@
+# Visualization Module Package

src/visualization/visualizer.py

@@ -0,0 +1,336 @@
+"""
+Visualization Module — Annotates images with detection results
+and action plan overlays using OpenCV.
+
+Provides two visualization modes:
+    1. **Detection overlay** — All detected objects with labels + confidence.
+    2. **Action overlay** — Source/destination bounding boxes, arrow, and steps.
+"""
+
+import logging
+from typing import Any
+
+import cv2
+import numpy as np
+
+logger = logging.getLogger(__name__)
+
+# ---------------------------------------------------------------------------
+# Color palette (BGR for OpenCV)
+# ---------------------------------------------------------------------------
+COLOR_GREEN = (0, 200, 0)
+COLOR_RED = (0, 0, 220)
+COLOR_YELLOW = (0, 220, 220)
+COLOR_CYAN = (220, 200, 0)
+COLOR_WHITE = (255, 255, 255)
+COLOR_BLACK = (0, 0, 0)
+COLOR_ORANGE = (0, 140, 255)
+COLOR_BLUE = (255, 100, 0)
+
+# Distinct palette for detection overlay
+_DETECTION_COLORS = [
+    COLOR_GREEN,
+    COLOR_CYAN,
+    COLOR_YELLOW,
+    COLOR_ORANGE,
+    COLOR_BLUE,
+    COLOR_RED,
+    (200, 0, 200),  # magenta
+    (100, 200, 100),  # light green
+]
+
+
+class ActionVisualizer:
+    """
+    Draws visual annotations on images for detection results and action plans.
+    """
+
+    # ------------------------------------------------------------------
+    # Detection Overlay
+    # ------------------------------------------------------------------
+
+    def draw_detections(
+        self,
+        image: np.ndarray,
+        detections: list[dict[str, Any]],
+    ) -> np.ndarray:
+        """
+        Draw all detected objects on the image with labels and confidence.
+
+        Args:
+            image: Input image (H, W, C) as NumPy array.
+            detections: List of detection dicts from the vision module.
+
+        Returns:
+            Annotated copy of the image.
+        """
+        if image is None or not isinstance(image, np.ndarray):
+            logger.error("Invalid image for detection visualization.")
+            return image if image is not None else np.zeros((400, 600, 3), dtype=np.uint8)
+
+        annotated = image.copy()
+
+        if not detections:
+            logger.info("No detections to visualize.")
+            self._put_centered_text(annotated, "No objects detected")
+            return annotated
+
+        for idx, det in enumerate(detections):
+            color = _DETECTION_COLORS[idx % len(_DETECTION_COLORS)]
+            bbox = det.get("bbox", [])
+            label = det.get("label", "unknown")
+            confidence = det.get("confidence", 0.0)
+
+            if len(bbox) != 4:
+                continue
+
+            x1, y1, x2, y2 = [int(c) for c in bbox]
+
+            # Draw bounding box
+            cv2.rectangle(annotated, (x1, y1), (x2, y2), color, 2)
+
+            # Label with confidence
+            text = f"{label} ({confidence:.2f})"
+            text_size = cv2.getTextSize(text, cv2.FONT_HERSHEY_SIMPLEX, 0.6, 2)[0]
+
+            # Background rectangle for text
+            cv2.rectangle(
+                annotated,
+                (x1, y1 - text_size[1] - 10),
+                (x1 + text_size[0] + 6, y1),
+                color,
+                -1,
+            )
+            cv2.putText(
+                annotated,
+                text,
+                (x1 + 3, y1 - 5),
+                cv2.FONT_HERSHEY_SIMPLEX,
+                0.6,
+                COLOR_WHITE,
+                2,
+            )
+
+        logger.info("Detection overlay drawn with %d objects.", len(detections))
+        return annotated
+
+    # ------------------------------------------------------------------
+    # Action Overlay
+    # ------------------------------------------------------------------
+
+    def visualize(
+        self,
+        image: np.ndarray,
+        action_plan: dict[str, Any],
+    ) -> np.ndarray:
+        """
+        Draw the action plan overlay: source/destination boxes, arrow,
+        and step labels.
+
+        Args:
+            image: Input image.
+            action_plan: Dict from the ActionPlanner containing
+                ``source_bbox``, ``dest_bbox``, ``steps``, etc.
+
+        Returns:
+            Annotated copy of the image.
+        """
+        if image is None or not isinstance(image, np.ndarray):
+            logger.error("Invalid image for action visualization.")
+            return image if image is not None else np.zeros((400, 600, 3), dtype=np.uint8)
+
+        annotated = image.copy()
+
+        source_bbox = action_plan.get("source_bbox")
+        dest_bbox = action_plan.get("dest_bbox")
+        steps = action_plan.get("steps", [])
+        status = action_plan.get("status", "unknown")
+
+        # Draw source bounding box (green)
+        if source_bbox and len(source_bbox) == 4:
+            self._draw_highlighted_box(
+                annotated, source_bbox, COLOR_GREEN, "SOURCE"
+            )
+
+        # Draw destination bounding box (red)
+        if dest_bbox and len(dest_bbox) == 4:
+            self._draw_highlighted_box(
+                annotated, dest_bbox, COLOR_RED, "DESTINATION"
+            )
+
+        # Draw arrow from source to destination
+        if source_bbox and dest_bbox:
+            self._draw_action_arrow(annotated, source_bbox, dest_bbox)
+
+        # Draw step annotations
+        self._draw_steps_overlay(annotated, steps, status)
+
+        logger.info("Action visualization drawn (status=%s).", status)
+        return annotated
+
+    # ------------------------------------------------------------------
+    # Drawing helpers
+    # ------------------------------------------------------------------
+
+    @staticmethod
+    def _draw_highlighted_box(
+        image: np.ndarray,
+        bbox: list[int],
+        color: tuple,
+        label: str,
+    ) -> None:
+        """Draw a highlighted bounding box with a corner-style frame."""
+        x1, y1, x2, y2 = [int(c) for c in bbox]
+        thickness = 3
+
+        # Semi-transparent overlay
+        overlay = image.copy()
+        cv2.rectangle(overlay, (x1, y1), (x2, y2), color, -1)
+        cv2.addWeighted(overlay, 0.15, image, 0.85, 0, image)
+
+        # Solid border
+        cv2.rectangle(image, (x1, y1), (x2, y2), color, thickness)
+
+        # Corner accents (L-shaped)
+        corner_len = min(30, (x2 - x1) // 4, (y2 - y1) // 4)
+        ct = thickness + 1
+        # Top-left
+        cv2.line(image, (x1, y1), (x1 + corner_len, y1), color, ct)
+        cv2.line(image, (x1, y1), (x1, y1 + corner_len), color, ct)
+        # Top-right
+        cv2.line(image, (x2, y1), (x2 - corner_len, y1), color, ct)
+        cv2.line(image, (x2, y1), (x2, y1 + corner_len), color, ct)
+        # Bottom-left
+        cv2.line(image, (x1, y2), (x1 + corner_len, y2), color, ct)
+        cv2.line(image, (x1, y2), (x1, y2 - corner_len), color, ct)
+        # Bottom-right
+        cv2.line(image, (x2, y2), (x2 - corner_len, y2), color, ct)
+        cv2.line(image, (x2, y2), (x2, y2 - corner_len), color, ct)
+
+        # Label above box
+        text_size = cv2.getTextSize(label, cv2.FONT_HERSHEY_SIMPLEX, 0.7, 2)[0]
+        cv2.rectangle(
+            image,
+            (x1, y1 - text_size[1] - 14),
+            (x1 + text_size[0] + 10, y1 - 2),
+            color,
+            -1,
+        )
+        cv2.putText(
+            image,
+            label,
+            (x1 + 5, y1 - 7),
+            cv2.FONT_HERSHEY_SIMPLEX,
+            0.7,
+            COLOR_WHITE,
+            2,
+        )
+
+    @staticmethod
+    def _draw_action_arrow(
+        image: np.ndarray,
+        source_bbox: list[int],
+        dest_bbox: list[int],
+    ) -> None:
+        """Draw a curved arrow from the source centre to the destination centre."""
+        sx = (source_bbox[0] + source_bbox[2]) // 2
+        sy = (source_bbox[1] + source_bbox[3]) // 2
+        dx = (dest_bbox[0] + dest_bbox[2]) // 2
+        dy = (dest_bbox[1] + dest_bbox[3]) // 2
+
+        # Draw a dashed path then a solid arrowhead
+        mid_x = (sx + dx) // 2
+        mid_y = min(sy, dy) - 40  # curve above
+
+        # Bezier-approximated curve via polyline
+        pts: list[tuple[int, int]] = []
+        for t_i in range(21):
+            t = t_i / 20.0
+            px = int((1 - t) ** 2 * sx + 2 * (1 - t) * t * mid_x + t ** 2 * dx)
+            py = int((1 - t) ** 2 * sy + 2 * (1 - t) * t * mid_y + t ** 2 * dy)
+            pts.append((px, py))
+
+        for i in range(len(pts) - 1):
+            # Dashed effect: draw every other segment
+            if i % 2 == 0:
+                cv2.line(image, pts[i], pts[i + 1], COLOR_ORANGE, 3)
+
+        # Arrowhead at destination
+        cv2.arrowedLine(
+            image, pts[-3], (dx, dy), COLOR_ORANGE, 3, tipLength=0.3
+        )
+
+        # "ACTION" label at midpoint
+        cv2.putText(
+            image,
+            "ACTION",
+            (mid_x - 30, mid_y - 10),
+            cv2.FONT_HERSHEY_SIMPLEX,
+            0.6,
+            COLOR_ORANGE,
+            2,
+        )
+
+    @staticmethod
+    def _draw_steps_overlay(
+        image: np.ndarray,
+        steps: list[str],
+        status: str,
+    ) -> None:
+        """Draw the steps list as a semi-transparent panel at the bottom."""
+        if not steps:
+            return
+
+        h, w = image.shape[:2]
+        line_height = 28
+        panel_height = len(steps) * line_height + 20
+        panel_top = h - panel_height
+
+        # Semi-transparent dark panel
+        overlay = image.copy()
+        cv2.rectangle(overlay, (0, panel_top), (w, h), COLOR_BLACK, -1)
+        cv2.addWeighted(overlay, 0.7, image, 0.3, 0, image)
+
+        # Status badge
+        status_color = (
+            COLOR_GREEN
+            if status == "success"
+            else COLOR_YELLOW if status == "partial" else COLOR_RED
+        )
+        cv2.putText(
+            image,
+            f"Status: {status.upper()}",
+            (10, panel_top + 20),
+            cv2.FONT_HERSHEY_SIMPLEX,
+            0.5,
+            status_color,
+            2,
+        )
+
+        # Steps text
+        for i, step in enumerate(steps):
+            y = panel_top + 20 + (i + 1) * line_height
+            # Truncate long steps
+            display_step = step if len(step) < 80 else step[:77] + "..."
+            cv2.putText(
+                image,
+                display_step,
+                (10, y),
+                cv2.FONT_HERSHEY_SIMPLEX,
+                0.5,
+                COLOR_WHITE,
+                1,
+            )
+
+    @staticmethod
+    def _put_centered_text(
+        image: np.ndarray,
+        text: str,
+        color: tuple = COLOR_WHITE,
+    ) -> None:
+        """Put centred text on the image."""
+        h, w = image.shape[:2]
+        text_size = cv2.getTextSize(text, cv2.FONT_HERSHEY_SIMPLEX, 1.0, 2)[0]
+        x = (w - text_size[0]) // 2
+        y = (h + text_size[1]) // 2
+        cv2.putText(image, text, (x, y), cv2.FONT_HERSHEY_SIMPLEX, 1.0, color, 2)

tests/__init__.py

@@ -0,0 +1 @@
+# Tests Package

tests/test_action_planner.py

@@ -0,0 +1,132 @@
+"""
+Tests for the Action Planning Module — ActionPlanner.
+"""
+
+import pytest
+
+from src.planner.action_planner import ActionPlanner
+
+
+class TestActionPlanner:
+    """Tests for ActionPlanner.plan() method."""
+
+    @pytest.fixture
+    def planner(self) -> ActionPlanner:
+        return ActionPlanner()
+
+    @pytest.fixture
+    def sample_detections(self) -> list[dict]:
+        return [
+            {"label": "bottle", "confidence": 0.92, "bbox": [100, 150, 200, 350]},
+            {"label": "box", "confidence": 0.85, "bbox": [400, 300, 550, 500]},
+            {"label": "cup", "confidence": 0.78, "bbox": [250, 200, 320, 300]},
+        ]
+
+    def test_plan_returns_required_keys(self, planner, sample_detections):
+        """Plan output must have all required keys."""
+        interpretation = {
+            "action": "move",
+            "source": "bottle",
+            "destination": "box",
+        }
+
+        result = planner.plan(sample_detections, interpretation)
+
+        assert "steps" in result
+        assert "source_bbox" in result
+        assert "dest_bbox" in result
+        assert "status" in result
+        assert "action" in result
+
+    def test_plan_success_status(self, planner, sample_detections):
+        """Plan should succeed when both source and destination match."""
+        interpretation = {
+            "action": "move",
+            "source": "bottle",
+            "destination": "box",
+        }
+
+        result = planner.plan(sample_detections, interpretation)
+
+        assert result["status"] == "success"
+        assert result["source_bbox"] == [100, 150, 200, 350]
+        assert result["dest_bbox"] == [400, 300, 550, 500]
+
+    def test_plan_partial_status(self, planner, sample_detections):
+        """Plan should be partial when destination doesn't match."""
+        interpretation = {
+            "action": "move",
+            "source": "bottle",
+            "destination": "nonexistent_object",
+        }
+
+        result = planner.plan(sample_detections, interpretation)
+
+        assert result["status"] == "partial"
+        assert result["source_bbox"] is not None
+        assert result["dest_bbox"] is None
+
+    def test_plan_failed_status(self, planner, sample_detections):
+        """Plan should fail when source doesn't match."""
+        interpretation = {
+            "action": "move",
+            "source": "nonexistent_object",
+            "destination": "box",
+        }
+
+        result = planner.plan(sample_detections, interpretation)
+
+        assert result["status"] == "failed"
+
+    def test_plan_generates_steps(self, planner, sample_detections):
+        """Plan should generate meaningful steps."""
+        interpretation = {
+            "action": "move",
+            "source": "bottle",
+            "destination": "box",
+        }
+
+        result = planner.plan(sample_detections, interpretation)
+
+        assert len(result["steps"]) >= 3
+        assert any("bottle" in step.lower() for step in result["steps"])
+        assert any("box" in step.lower() for step in result["steps"])
+
+    def test_plan_no_destination(self, planner, sample_detections):
+        """Plan should work when destination is None (e.g. 'inspect')."""
+        interpretation = {
+            "action": "inspect",
+            "source": "cup",
+            "destination": None,
+        }
+
+        result = planner.plan(sample_detections, interpretation)
+
+        assert result["status"] == "success"
+        assert result["dest_bbox"] is None
+
+    def test_plan_empty_detections(self, planner):
+        """Plan should fail gracefully with no detections."""
+        interpretation = {
+            "action": "move",
+            "source": "bottle",
+            "destination": "box",
+        }
+
+        result = planner.plan([], interpretation)
+
+        assert result["status"] == "failed"
+
+    def test_plan_with_synonym(self, planner, sample_detections):
+        """Plan should resolve synonyms via ObjectMapper (mug → cup)."""
+        interpretation = {
+            "action": "pick_up",
+            "source": "mug",
+            "destination": None,
+        }
+
+        result = planner.plan(sample_detections, interpretation)
+
+        # "mug" should map to "cup" via the synonym mapper
+        assert result["status"] == "success"
+        assert result["source_bbox"] == [250, 200, 320, 300]

tests/test_detector.py

@@ -0,0 +1,60 @@
+"""
+Tests for the Vision Module — ObjectDetector.
+"""
+
+import numpy as np
+import pytest
+
+
+class TestObjectDetector:
+    """Tests for ObjectDetector.detect() method."""
+
+    def test_detect_returns_list(self):
+        """Detection should return a list."""
+        from src.vision.detector import ObjectDetector
+
+        detector = ObjectDetector(model_name="yolov8n.pt", confidence=0.25)
+
+        # Create a synthetic test image (solid blue, 480x640)
+        image = np.zeros((480, 640, 3), dtype=np.uint8)
+        image[:, :] = (200, 100, 50)
+
+        result = detector.detect(image)
+        assert isinstance(result, list), "detect() must return a list"
+
+    def test_detect_result_schema(self):
+        """Each detection should have label, confidence, and bbox keys."""
+        from src.vision.detector import ObjectDetector
+
+        detector = ObjectDetector(model_name="yolov8n.pt", confidence=0.25)
+
+        # Use a real-ish image to maybe get detections
+        image = np.random.randint(0, 255, (480, 640, 3), dtype=np.uint8)
+        result = detector.detect(image)
+
+        for det in result:
+            assert "label" in det, "Detection must have 'label'"
+            assert "confidence" in det, "Detection must have 'confidence'"
+            assert "bbox" in det, "Detection must have 'bbox'"
+            assert isinstance(det["label"], str)
+            assert isinstance(det["confidence"], float)
+            assert isinstance(det["bbox"], list)
+            assert len(det["bbox"]) == 4
+
+    def test_detect_invalid_image_raises(self):
+        """Passing None should raise ValueError."""
+        from src.vision.detector import ObjectDetector
+
+        detector = ObjectDetector(model_name="yolov8n.pt", confidence=0.25)
+
+        with pytest.raises(ValueError):
+            detector.detect(None)
+
+    def test_detect_1d_image_raises(self):
+        """Passing a 1D array should raise ValueError."""
+        from src.vision.detector import ObjectDetector
+
+        detector = ObjectDetector(model_name="yolov8n.pt", confidence=0.25)
+
+        with pytest.raises(ValueError):
+            detector.detect(np.array([1, 2, 3]))

tests/test_object_mapper.py

@@ -0,0 +1,71 @@
+"""
+Tests for the Object Mapper — synonym normalization and fuzzy matching.
+"""
+
+import pytest
+
+from src.utils.object_mapper import (
+    normalize_label,
+    find_best_match,
+    SYNONYM_MAP,
+)
+
+
+class TestNormalizeLabel:
+    """Tests for normalize_label()."""
+
+    def test_known_synonym(self):
+        assert normalize_label("mug") == "cup"
+
+    def test_known_synonym_case_insensitive(self):
+        assert normalize_label("MUG") == "cup"
+
+    def test_unknown_label_passthrough(self):
+        assert normalize_label("bottle") == "bottle"
+
+    def test_whitespace_stripped(self):
+        assert normalize_label("  mug  ") == "cup"
+
+    def test_container_to_bottle(self):
+        assert normalize_label("container") == "bottle"
+
+    def test_smartphone_to_cell_phone(self):
+        assert normalize_label("smartphone") == "cell phone"
+
+
+class TestFindBestMatch:
+    """Tests for find_best_match()."""
+
+    @pytest.fixture
+    def detected_labels(self) -> list[str]:
+        return ["bottle", "cup", "box", "person", "laptop"]
+
+    def test_exact_match(self, detected_labels):
+        result = find_best_match("bottle", detected_labels)
+        assert result == "bottle"
+
+    def test_synonym_match(self, detected_labels):
+        """'mug' should resolve to 'cup' via synonym normalization."""
+        result = find_best_match("mug", detected_labels)
+        assert result == "cup"
+
+    def test_container_matches_bottle(self, detected_labels):
+        result = find_best_match("container", detected_labels)
+        assert result == "bottle"
+
+    def test_substring_match(self, detected_labels):
+        result = find_best_match("lap", detected_labels)
+        assert result == "laptop"
+
+    def test_no_match_returns_none(self, detected_labels):
+        result = find_best_match("xyznonexistent", detected_labels)
+        assert result is None
+
+    def test_empty_query(self, detected_labels):
+        result = find_best_match("", detected_labels)
+        # Empty normalizes to empty, no match expected
+        assert result is None or isinstance(result, str)
+
+    def test_empty_detected_labels(self):
+        result = find_best_match("bottle", [])
+        assert result is None

tests/test_reasoner.py

@@ -0,0 +1,112 @@
+"""
+Tests for the Language Reasoning Module — CommandReasoner.
+
+Uses mocked OpenAI client to test interpretation logic without API calls.
+"""
+
+import json
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+from src.language.reasoner import CommandReasoner
+
+
+class TestCommandReasoner:
+    """Tests for CommandReasoner.interpret() method."""
+
+    @pytest.fixture
+    def sample_detections(self) -> list[dict]:
+        """Sample detection results for testing."""
+        return [
+            {"label": "bottle", "confidence": 0.92, "bbox": [100, 150, 200, 350]},
+            {"label": "box", "confidence": 0.85, "bbox": [400, 300, 550, 500]},
+            {"label": "cup", "confidence": 0.78, "bbox": [250, 200, 320, 300]},
+        ]
+
+    def test_fallback_mode_without_api_key(self, sample_detections):
+        """Reasoner without API key should use fallback mode."""
+        reasoner = CommandReasoner(provider="openai", api_key="", model="gpt-4o")
+
+        result = reasoner.interpret(
+            "Move the bottle to the box", sample_detections
+        )
+
+        assert isinstance(result, dict)
+        assert "action" in result
+        assert "source" in result
+        assert "destination" in result
+
+    def test_fallback_extracts_action_verb(self, sample_detections):
+        """Fallback mode should detect the action verb from the command."""
+        reasoner = CommandReasoner(provider="openai", api_key="", model="gpt-4o")
+
+        result = reasoner.interpret(
+            "inspect the cup carefully", sample_detections
+        )
+
+        assert result["action"] == "inspect"
+
+    def test_fallback_matches_labels(self, sample_detections):
+        """Fallback mode should match detected labels from the command."""
+        reasoner = CommandReasoner(provider="openai", api_key="", model="gpt-4o")
+
+        result = reasoner.interpret(
+            "Move the bottle to the box", sample_detections
+        )
+
+        assert result["source"] == "bottle"
+        assert result["destination"] == "box"
+
+    def test_empty_command_returns_error(self, sample_detections):
+        """Empty command should return an error result."""
+        reasoner = CommandReasoner(provider="openai", api_key="", model="gpt-4o")
+
+        result = reasoner.interpret("", sample_detections)
+
+        assert result["action"] == "error"
+        assert "error" in result
+
+    @patch("src.language.reasoner.CommandReasoner._init_client")
+    def test_llm_interpretation(self, mock_init, sample_detections):
+        """Test successful LLM-based interpretation with mocked client."""
+        reasoner = CommandReasoner(provider="openai", api_key="test-key", model="gpt-4o")
+
+        # Mock the OpenAI client response
+        mock_response = MagicMock()
+        mock_response.choices = [MagicMock()]
+        mock_response.choices[0].message.content = json.dumps(
+            {"action": "move", "source": "bottle", "destination": "box"}
+        )
+
+        mock_client = MagicMock()
+        mock_client.chat.completions.create.return_value = mock_response
+        reasoner.client = mock_client
+
+        result = reasoner.interpret(
+            "Move the bottle to the box", sample_detections
+        )
+
+        assert result["action"] == "move"
+        assert result["source"] == "bottle"
+        assert result["destination"] == "box"
+
+    def test_parse_response_with_markdown_fences(self):
+        """Parser should handle markdown code fences from LLM."""
+        reasoner = CommandReasoner(provider="openai", api_key="", model="gpt-4o")
+
+        raw = '```json\n{"action": "pick_up", "source": "cup", "destination": null}\n```'
+        result = reasoner._parse_response(raw)
+
+        assert result["action"] == "pick_up"
+        assert result["source"] == "cup"
+        assert result["destination"] is None
+
+    def test_parse_response_invalid_json(self):
+        """Parser should return error dict for invalid JSON."""
+        reasoner = CommandReasoner(provider="openai", api_key="", model="gpt-4o")
+
+        result = reasoner._parse_response("this is not json")
+
+        assert result["action"] == "unknown"
+        assert "error" in result

tests/test_visualizer.py

@@ -0,0 +1,106 @@
+"""
+Tests for the Visualization Module — ActionVisualizer.
+"""
+
+import numpy as np
+import pytest
+
+from src.visualization.visualizer import ActionVisualizer
+
+
+class TestActionVisualizer:
+    """Tests for ActionVisualizer methods."""
+
+    @pytest.fixture
+    def visualizer(self) -> ActionVisualizer:
+        return ActionVisualizer()
+
+    @pytest.fixture
+    def test_image(self) -> np.ndarray:
+        """A 480x640 synthetic test image."""
+        return np.zeros((480, 640, 3), dtype=np.uint8)
+
+    @pytest.fixture
+    def sample_detections(self) -> list[dict]:
+        return [
+            {"label": "bottle", "confidence": 0.92, "bbox": [100, 150, 200, 350]},
+            {"label": "box", "confidence": 0.85, "bbox": [400, 300, 550, 500]},
+        ]
+
+    @pytest.fixture
+    def sample_action_plan(self) -> dict:
+        return {
+            "action": "move",
+            "steps": [
+                "Step 1: Locate 'bottle'",
+                "Step 2: Locate 'box'",
+                "Step 3: Move 'bottle' → 'box'",
+                "Step 4: Verify action completion",
+            ],
+            "source_bbox": [100, 150, 200, 350],
+            "dest_bbox": [400, 300, 550, 500],
+            "status": "success",
+        }
+
+    def test_draw_detections_returns_image(
+        self, visualizer, test_image, sample_detections
+    ):
+        """draw_detections should return an image of the same shape."""
+        result = visualizer.draw_detections(test_image, sample_detections)
+
+        assert isinstance(result, np.ndarray)
+        assert result.shape == test_image.shape
+
+    def test_draw_detections_modifies_image(
+        self, visualizer, test_image, sample_detections
+    ):
+        """draw_detections should modify the image (not return a blank copy)."""
+        result = visualizer.draw_detections(test_image, sample_detections)
+
+        # The annotated image should differ from the original
+        assert not np.array_equal(result, test_image)
+
+    def test_draw_detections_empty_list(self, visualizer, test_image):
+        """draw_detections with empty list should still return an image."""
+        result = visualizer.draw_detections(test_image, [])
+
+        assert isinstance(result, np.ndarray)
+        assert result.shape == test_image.shape
+
+    def test_visualize_returns_image(
+        self, visualizer, test_image, sample_action_plan
+    ):
+        """visualize should return an image of the same shape."""
+        result = visualizer.visualize(test_image, sample_action_plan)
+
+        assert isinstance(result, np.ndarray)
+        assert result.shape == test_image.shape
+
+    def test_visualize_modifies_image(
+        self, visualizer, test_image, sample_action_plan
+    ):
+        """visualize should modify the image."""
+        result = visualizer.visualize(test_image, sample_action_plan)
+
+        assert not np.array_equal(result, test_image)
+
+    def test_visualize_with_no_bboxes(self, visualizer, test_image):
+        """visualize should handle missing bounding boxes gracefully."""
+        plan = {
+            "action": "inspect",
+            "steps": ["Step 1: Inspect object"],
+            "source_bbox": None,
+            "dest_bbox": None,
+            "status": "failed",
+        }
+
+        result = visualizer.visualize(test_image, plan)
+
+        assert isinstance(result, np.ndarray)
+        assert result.shape == test_image.shape
+
+    def test_draw_detections_invalid_image(self, visualizer):
+        """draw_detections with None should return a fallback image."""
+        result = visualizer.draw_detections(None, [])
+
+        assert isinstance(result, np.ndarray)