merge with main

agent/__init__.py

@@ -0,0 +1,4 @@
+from .orchestrator import run_agent
+from .tools import AGENT_TOOLS, dispatch_tool
+
+__all__ = ["run_agent", "AGENT_TOOLS", "dispatch_tool"]

agent/orchestrator.py

@@ -0,0 +1,166 @@
+"""
+Agentic orchestration loop for the VL model.
+
+Calls `InferenceClient.chat_completion` with the three agent tools defined in
+`agent/tools.py`.  The loop continues as long as the model emits tool calls.
+It stops (and returns a final string) when:
+  - The model calls `final_output`  -> return the `answer` argument
+  - The model calls `abort`         -> return "Aborted: {reason}"
+  - The model returns plain content with no tool calls
+  - `max_tool_rounds` is reached    -> return the last assistant content
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from huggingface_hub import InferenceClient
+
+from .tools import AGENT_TOOLS, dispatch_tool
+
+DEFAULT_MAX_TOOL_ROUNDS = 10
+
+
+def run_agent(
+    messages: list[dict[str, Any]],
+    client: InferenceClient,
+    model: str,
+    max_tokens: int = 512,
+    temperature: float = 0.7,
+    top_p: float = 0.95,
+    max_tool_rounds: int = DEFAULT_MAX_TOOL_ROUNDS,
+) -> str:
+    """
+    Run the agentic loop and return the final answer as a string.
+
+    Parameters
+    ----------
+    messages:
+        Full conversation so far, including the system message, all history,
+        and the latest user message (which may contain an image as a multimodal
+        content list).
+    client:
+        An authenticated `InferenceClient` instance.
+    model:
+        HF model ID to use for inference.
+    max_tokens:
+        Maximum tokens per completion call.
+    temperature:
+        Sampling temperature.
+    top_p:
+        Nucleus sampling top-p.
+    max_tool_rounds:
+        Hard cap on how many tool-calling rounds are allowed before the loop
+        gives up and returns whatever the model last said.
+    """
+    messages = list(messages)  # work on a local copy
+
+    for _round in range(max_tool_rounds + 1):
+        response = client.chat_completion(
+            messages=messages,
+            model=model,
+            tools=AGENT_TOOLS,
+            tool_choice="auto",
+            max_tokens=max_tokens,
+            temperature=temperature,
+            top_p=top_p,
+            stream=False,
+        )
+
+        choice = response.choices[0]
+        msg = choice.message
+
+        tool_calls = getattr(msg, "tool_calls", None) or []
+
+        # ------------------------------------------------------------------ #
+        # No tool calls → plain text answer, we're done                       #
+        # ------------------------------------------------------------------ #
+        if not tool_calls:
+            return msg.content or ""
+
+        # ------------------------------------------------------------------ #
+        # There are tool calls → check for terminal tools first               #
+        # ------------------------------------------------------------------ #
+        for tc in tool_calls:
+            fn = tc.function
+            name = fn.name
+            raw_args = fn.arguments or "{}"
+            args = raw_args if isinstance(raw_args, dict) else _safe_parse(raw_args)
+
+            if name == "final_output":
+                return args.get("answer", msg.content or "")
+
+            if name == "abort":
+                reason = args.get("reason", "")
+                return f"Aborted: {reason}" if reason else "Task aborted."
+
+        # ------------------------------------------------------------------ #
+        # Non-terminal tool calls → execute each one and feed results back    #
+        # ------------------------------------------------------------------ #
+        messages.append(_assistant_tool_call_message(msg))
+
+        for tc in tool_calls:
+            fn = tc.function
+            name = fn.name
+            raw_args = fn.arguments or "{}"
+            args = raw_args if isinstance(raw_args, dict) else _safe_parse(raw_args)
+
+            result = dispatch_tool(name, args)
+
+            messages.append(
+                {
+                    "role": "tool",
+                    "content": result,
+                    "tool_call_id": tc.id,
+                }
+            )
+
+    # Max rounds exhausted — return the last assistant content if any
+    last_assistant = next(
+        (m["content"] for m in reversed(messages) if m.get("role") == "assistant"),
+        "I was unable to complete the task within the allowed number of steps.",
+    )
+    return last_assistant or "I was unable to complete the task within the allowed number of steps."
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+def _safe_parse(raw: str) -> dict:
+    """Parse a JSON string into a dict, returning {} on failure."""
+    try:
+        return json.loads(raw)
+    except (json.JSONDecodeError, TypeError):
+        return {}
+
+
+def _assistant_tool_call_message(msg: Any) -> dict:
+    """
+    Re-serialise the assistant message that contains tool_calls into the plain
+    dict format expected when appended back to `messages`.
+    """
+    tool_calls_payload = []
+    for tc in msg.tool_calls or []:
+        fn = tc.function
+        tool_calls_payload.append(
+            {
+                "id": tc.id,
+                "type": "function",
+                "function": {
+                    "name": fn.name,
+                    "arguments": (
+                        fn.arguments
+                        if isinstance(fn.arguments, str)
+                        else json.dumps(fn.arguments)
+                    ),
+                },
+            }
+        )
+
+    return {
+        "role": "assistant",
+        "content": msg.content or "",
+        "tool_calls": tool_calls_payload,
+    }

agent/tools.py

@@ -0,0 +1,149 @@
+"""
+Tool definitions (HF/OpenAI schema) and dispatcher for the VL agent.
+
+Three tools:
+  - search_web:    search the internet and return snippets
+  - final_output:  signal the agent loop to stop and return an answer
+  - abort:         signal the agent loop to stop and report a failure/reason
+"""
+
+import json
+
+# ---------------------------------------------------------------------------
+# Tool schemas (passed to InferenceClient.chat_completion(tools=...))
+# ---------------------------------------------------------------------------
+
+AGENT_TOOLS = [
+    {
+        "type": "function",
+        "function": {
+            "name": "search_web",
+            "description": (
+                "Search the internet for up-to-date information. "
+                "Use this when you need current facts, news, or data that you are "
+                "not confident about from your training knowledge."
+            ),
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "query": {
+                        "type": "string",
+                        "description": "The search query to look up on the web.",
+                    }
+                },
+                "required": ["query"],
+            },
+        },
+    },
+    {
+        "type": "function",
+        "function": {
+            "name": "final_output",
+            "description": (
+                "Deliver the final answer to the user. "
+                "Call this once you have gathered enough information and are ready "
+                "to give a complete, accurate response. The 'answer' field will be "
+                "shown directly to the user."
+            ),
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "answer": {
+                        "type": "string",
+                        "description": "The complete final answer to present to the user.",
+                    }
+                },
+                "required": ["answer"],
+            },
+        },
+    },
+    {
+        "type": "function",
+        "function": {
+            "name": "abort",
+            "description": (
+                "Abort the current task when it cannot be completed. "
+                "Use this if the task is impossible, unsafe, or the user asked to stop."
+            ),
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "reason": {
+                        "type": "string",
+                        "description": "Explanation of why the task is being aborted.",
+                    }
+                },
+                "required": [],
+            },
+        },
+    },
+]
+
+
+# ---------------------------------------------------------------------------
+# Tool implementations
+# ---------------------------------------------------------------------------
+
+def _search_web(query: str, max_results: int = 5) -> str:
+    """Run a DuckDuckGo text search and return formatted snippets."""
+    try:
+        from duckduckgo_search import DDGS
+
+        results = []
+        with DDGS() as ddgs:
+            for r in ddgs.text(query, max_results=max_results):
+                title = r.get("title", "")
+                body = r.get("body", "")
+                href = r.get("href", "")
+                results.append(f"**{title}**\n{body}\nSource: {href}")
+
+        if not results:
+            return "No results found for the given query."
+
+        return "\n\n---\n\n".join(results)
+
+    except Exception as exc:  # noqa: BLE001
+        return f"Search failed: {exc}"
+
+
+def _final_output(answer: str) -> str:
+    """No-op implementation; the orchestrator reads 'answer' directly."""
+    return "Answer delivered."
+
+
+def _abort(reason: str = "") -> str:
+    """No-op implementation; the orchestrator reads 'reason' directly."""
+    return f"Aborted: {reason}" if reason else "Task aborted."
+
+
+# ---------------------------------------------------------------------------
+# Dispatcher
+# ---------------------------------------------------------------------------
+
+def dispatch_tool(tool_name: str, arguments: dict | str) -> str:
+    """
+    Call the named tool with the given arguments and return the result string.
+
+    `arguments` may arrive as a JSON string (from the model) or already as a dict.
+    """
+    if isinstance(arguments, str):
+        try:
+            arguments = json.loads(arguments)
+        except json.JSONDecodeError:
+            arguments = {}
+
+    if tool_name == "search_web":
+        query = arguments.get("query", "")
+        if not query:
+            return "Error: 'query' parameter is required for search_web."
+        return _search_web(query)
+
+    if tool_name == "final_output":
+        answer = arguments.get("answer", "")
+        return _final_output(answer)
+
+    if tool_name == "abort":
+        reason = arguments.get("reason", "")
+        return _abort(reason)
+
+    return f"Error: unknown tool '{tool_name}'."

app.py

@@ -1,5 +1,6 @@
 import base64
-import io
+from pathlib import Path
+from typing import Any
 
 import gradio as gr
 from huggingface_hub import InferenceClient
@@ -18,71 +19,124 @@ def _image_to_data_url(image_path: str, max_side: int = 1120, quality: int = 85)
         img.save(buf, format="JPEG", quality=quality)
     return "data:image/jpeg;base64," + base64.b64encode(buf.getvalue()).decode()
 
+from agent import run_agent
+
+MODEL = "Qwen/Qwen3-VL-30B-A3B-Thinking"
+
+DEFAULT_SYSTEM = (
+    "You are a helpful, multimodal AI assistant. "
+    "You can analyse images the user uploads and answer questions about them. "
+    "When you need up-to-date information from the internet, use the search_web tool. "
+    "Once you have a complete answer, call final_output with the full answer. "
+    "If a task is impossible or unsafe, call abort with a brief reason."
+)
+
+
+def _image_to_data_url(image_path: str, max_side: int = 1120, quality: int = 85) -> str:
+    """
+    Convert a local image file to a base64 data URL, resizing it first so the
+    payload stays within the HF router's request-size limit.
+
+    Images are downscaled so their longest side is at most `max_side` pixels,
+    then saved as JPEG at `quality` to keep the base64 size small.
+    """
+    from PIL import Image
+    import io
+
+    with Image.open(image_path) as img:
+        img = img.convert("RGB")
+        w, h = img.size
+        if max(w, h) > max_side:
+            scale = max_side / max(w, h)
+            img = img.resize((int(w * scale), int(h * scale)), Image.LANCZOS)
+
+        buf = io.BytesIO()
+        img.save(buf, format="JPEG", quality=quality)
+        data = base64.b64encode(buf.getvalue()).decode("utf-8")
+
+    return f"data:image/jpeg;base64,{data}"
+
+
+def _build_user_content(text: str, image_path: str | None) -> Any:
+    """
+    Build the `content` field for a user message.
+
+    Returns a plain string when there is no image, or a list of content parts
+    (text + image_url) when an image is present.
+    """
+    if not image_path:
+        return text or ""
+
+    parts: list[dict] = []
+    if text:
+        parts.append({"type": "text", "text": text})
+    parts.append(
+        {
+            "type": "image_url",
+            "image_url": {"url": _image_to_data_url(image_path)},
+        }
+    )
+    return parts
+
 
 def respond(
-    message,
+    message: str,
     history: list[dict],
-    system_message,
-    max_tokens,
-    temperature,
-    top_p,
+    image: str | None,
+    system_message: str,
+    max_tokens: int,
+    temperature: float,
+    top_p: float,
     hf_token: gr.OAuthToken,
 ):
-    client = InferenceClient(token=hf_token.token, model="Qwen/Qwen3-VL-30B-A3B-Thinking")
+    client = InferenceClient(token=hf_token.token, model=MODEL)
 
-    messages = [{"role": "system", "content": system_message}]
+    # Build the full message list: system + history + new user turn
+    messages: list[dict] = [{"role": "system", "content": system_message}]
     messages.extend(history)
 
-    # message is a dict {"text": str, "files": [path, ...]} from MultimodalTextbox
-    text = message.get("text", "") if isinstance(message, dict) else message
-    files = message.get("files", []) if isinstance(message, dict) else []
-
-    if files:
-        content = []
-        if text:
-            content.append({"type": "text", "text": text})
-        for f in files:
-            content.append({"type": "image_url", "image_url": {"url": _image_to_data_url(f)}})
-    else:
-        content = text
+    user_content = _build_user_content(message, image)
+    messages.append({"role": "user", "content": user_content})
 
-    messages.append({"role": "user", "content": content})
-
-    response = ""
-    for chunk in client.chat_completion(
-        messages,
+    # Run the agentic loop and get back the final answer string
+    answer = run_agent(
+        messages=messages,
+        client=client,
+        model=MODEL,
         max_tokens=max_tokens,
-        stream=True,
         temperature=temperature,
         top_p=top_p,
-    ):
-        choices = chunk.choices
-        if choices and choices[0].delta.content:
-            response += choices[0].delta.content
-        yield response
-
-
-chatbot = gr.ChatInterface(
-    respond,
-    multimodal=True,
-    additional_inputs=[
-        gr.Textbox(value="You are a friendly Chatbot.", label="System message"),
-        gr.Slider(minimum=1, maximum=16384, value=8192, step=1, label="Max new tokens"),
-        gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"),
-        gr.Slider(
-            minimum=0.1,
-            maximum=1.0,
-            value=0.95,
-            step=0.05,
-            label="Top-p (nucleus sampling)",
-        ),
-    ],
-)
+    )
+
+    # Yield in chunks to preserve Gradio's streaming UX
+    chunk_size = 8
+    partial = ""
+    for i in range(0, len(answer), chunk_size):
+        partial += answer[i : i + chunk_size]
+        yield partial
+
 
 with gr.Blocks() as demo:
     with gr.Sidebar():
         gr.LoginButton()
-    chatbot.render()
+        image_input = gr.Image(label="Upload image", type="filepath")
+
+    gr.ChatInterface(
+        respond,
+        additional_inputs=[
+            image_input,
+            gr.Textbox(value=DEFAULT_SYSTEM, label="System message"),
+            gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max new tokens"),
+            gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"),
+            gr.Slider(
+                minimum=0.1,
+                maximum=1.0,
+                value=0.95,
+                step=0.05,
+                label="Top-p (nucleus sampling)",
+            ),
+        ],
+    )
 
 
 if __name__ == "__main__":

requirements.txt

@@ -0,0 +1,3 @@
+gradio>=5.0.0
+huggingface_hub>=0.25.0
+duckduckgo-search>=6.0.0