Completed some demos from huggingface tutorials.

app.py

@@ -1,77 +1,148 @@
+import json
 import gradio as gr
-from huggingface_hub import InferenceClient
+import os
+from huggingface_hub import InferenceClient, login
+from image_agent import init_stored_image_agent, stored_images
+from dynamic_image_agent import main
+from mulitagents import define_multi_agent
+from nsfw_detection import classify_image_if_nsfw
 
+login(os.environ.get('HF_TOKEN'))
+os.environ["OPENAI_API_KEY"] = os.environ.get('OPENAI_API_KEY')
 
+# --- Global state ---
+active_agent = None  # will store which agent is currently selected
+
+def init_and_extend_messages(system_msg: object, history: list[dict[str, str]]):
+    messages = [system_msg]
+    messages.extend(history)
+    return messages
+
+# --- Respond function ---
 def respond(
     message,
     history: list[dict[str, str]],
     system_message,
     max_tokens,
     temperature,
-    top_p,
-    hf_token: gr.OAuthToken,
+    top_p
 ):
-    """
-    For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference
-    """
-    client = InferenceClient(token=hf_token.token, model="openai/gpt-oss-20b")
-
-    messages = [{"role": "system", "content": system_message}]
-
-    messages.extend(history)
-
-    messages.append({"role": "user", "content": message})
-
-    response = ""
-
-    for message in client.chat_completion(
-        messages,
-        max_tokens=max_tokens,
-        stream=True,
-        temperature=temperature,
-        top_p=top_p,
-    ):
-        choices = message.choices
-        token = ""
-        if len(choices) and choices[0].delta.content:
-            token = choices[0].delta.content
-
-        response += token
-        yield response
-
-
-"""
-For information on how to customize the ChatInterface, peruse the gradio docs: https://www.gradio.app/docs/chatinterface
-"""
-
-# Function to clear chat
-def clear_chat():
-    return gr.update(value=[])
-    
+    """Routes the user message to the active agent."""
+    global active_agent
+
+    if not active_agent:
+        return "⚠️ Please select an agent before chatting."
+
+
+
+    try:
+        # Route message to correct agent
+        if active_agent == "stored":
+            response = init_stored_image_agent().run(
+                f"""
+                {message}
+                """,
+                images=stored_images
+            )
+        elif active_agent == "dynamic":
+            json_response = main()
+            print("JSON RESPONSE:", json_response)
+            if isinstance(json_response, str):
+                return json_response
+            else:
+                try:
+                    response = json.dumps(json_response, indent=4)
+                    return response
+                except (json.JSONDecodeError, TypeError):
+                    # JSONDecodeError for invalid JSON format in a string
+                    # TypeError if the input is not a string or bytes-like object
+                    print("Error parsing json response:", json_response)
+                    return "Error generating response"
+        elif active_agent == "multi":
+            manager_agent = define_multi_agent()
+            manager_agent.visualize()
+
+            json_response = manager_agent.run(f"""{message}""")
+
+            manager_agent.python_executor.state["fig"]
+            if isinstance(json_response, str):
+                response = json_response
+                return response
+            else:
+                try:
+                    response = json.dumps(json_response, indent=4)
+                    return response
+                except (json.JSONDecodeError, TypeError):
+                    # JSONDecodeError for invalid JSON format in a string
+                    # TypeError if the input is not a string or bytes-like object
+                    print("Error parsing json response:", json_response)
+                    return "Error generating response"
+        elif active_agent == "nsfw check":
+            json_response = classify_image_if_nsfw(message)
+            response = json.dumps(json_response, indent=4)
+            return response
+        else:
+            response = f"Unknown agent selected."
+    except Exception as e:
+        print("Exception:", str(e))
+
+        response = f"⚠️ Error: {e}"
+
+    return response
+
+
+# --- Button Handlers ---
+def use_stored_image_agent():
+    global active_agent
+    active_agent = "stored"
+    return "✅ Switched to Stored Image Agent."
+
+
+def use_dynamic_image_agent():
+    global active_agent
+    active_agent = "dynamic"
+    return "✅ Switched to Dynamic Image Agent."
+
+
+def use_multi_agent():
+    global active_agent
+    active_agent = "multi"
+    return "✅ Switched to Multi-Agent mode."
+
+
+def use_nsfw_check():
+    global active_agent
+    active_agent = "nsfw check"
+    classify_image_if_nsfw("https://static.api4.ai/api4.ai/nsfw/demo-pic-1.jpg")
+    return "✅ Switched to NSFW Check mode."
+
+
+# --- Chat Interface ---
 chatbot = gr.ChatInterface(
-    respond,
+    fn=respond,
     type="messages",
-    additional_inputs=[
-        gr.Textbox(value="You are a friendly Chatbot.", label="System message"),
-    ],
+    additional_inputs=[]
 )
 
+# --- Layout ---
 with gr.Blocks() as demo:
-    with gr.Sidebar():
-        gr.LoginButton()
 
     chatbot.render()
-    
-    # Define a button outside ChatInterface (but visually below)
-    clear_button = gr.Button("🧹 Clear Chat", variant="secondary")
-
-    # Attach event listener to the button
-    clear_button.click(
-        fn=clear_chat,
-        inputs=None,
-        outputs=chatbot
-    )
-
-
+    gr.Markdown("### 🧩 Choose an Agent:")
+    with gr.Row():
+        stored_img_button = gr.Button("Checked Stored Superheros", variant="secondary")
+        dynamic_img_button = gr.Button("Dynamically look for superheros", variant="primary")
+        multi_agent_button = gr.Button("Search superhero's using multiple agents", variant="secondary")
+        check_nsfw_button = gr.Button("NSFW Check on Image", variant="stop")
+
+    # Display agent switch confirmation message
+    status_box = gr.Textbox(label="Agent Status", interactive=False)
+
+    stored_img_button.click(fn=use_stored_image_agent, inputs=None, outputs=status_box)
+    dynamic_img_button.click(fn=use_dynamic_image_agent, inputs=None, outputs=status_box)
+    multi_agent_button.click(fn=use_multi_agent, inputs=None, outputs=status_box)
+    check_nsfw_button.click(fn= use_nsfw_check, inputs=None, outputs=status_box)
+
+# --- Run app ---
 if __name__ == "__main__":
     demo.launch()

debug_image.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9feb6223827c2f855913bfe56f43b8463ea61c448dede1765dd3cf92d1e1d549
+size 92

dynamic_image_agent.py

@@ -1,18 +1,19 @@
-from langchain_community.tools.ddg_search.tool import DuckDuckGoSearchTool
-from io import BytesIO
-from PIL import Image
+import argparse
+import os
+import helium
 from time import sleep
+from datetime import datetime
 
-import helium
 from dotenv import load_dotenv
+from PIL import Image
 from selenium import webdriver
 from selenium.webdriver.common.by import By
 from selenium.webdriver.common.keys import Keys
 
-from smolagents import tool, CodeAgent, OpenAIServerModel
+from smolagents import tool, CodeAgent, DuckDuckGoSearchTool, Model, InferenceClientModel
 from smolagents.agents import ActionStep
 from smolagents.cli import load_model
-
+from io import BytesIO
 
 agent_request = """
     I am alfred, the butler of wayne manor, responsible for verifying the identity of guests at party. A superhero has arrived at the entrance claiming to be Wonderwomen, but I need to confirm if she is who she says she is.
@@ -74,35 +75,40 @@ def close_popups() -> str:
     """
     webdriver.ActionChains(driver).send_keys(Keys.ESCAPE).perform()
 
-def save_screenshot(step_log: ActionStep, agent: CodeAgent) -> None:
-    sleep(1.0)
-    driver = helium.get_driver()
-    current_step = step_log.step_number
-    if driver is not None:
-        for step_logs in agent.logs:
-            if isinstance(step_log, ActionStep) and step_log.step_number <= current_step - 2:
-                step_logs.observation_images = None
-        png_bytes = driver.get_screenshot_as_png()
-        image = Image.open(BytesIO(png_bytes))
-        print(f"Captured a browser screenshot: {image.size} pixels")
-        step_log.observation_images = image = [image.copy()]
-
-    # Update observations with current URL
-    url_info = f"Current url: {driver.current_url}"
-    step_log.observations = url_info if step_logs.observations is None else step_log.observations + "\n" + url_info
-    return
-
-model = OpenAIServerModel(model="gpt-4o")
-
-def initialize_agent(model: OpenAIServerModel):
-    return CodeAgent(
-        tools=[DuckDuckGoSearchTool(), go_back, close_popups, search_item_ctrl_f, save_screenshot],
-        model=model,
-        additional_authorized_imports=["helium"],
-        step_callbacks=[save_screenshot],
-        max_steps=20,
-        verbosity_level=2
-    )
+
+def save_screenshot(step):
+    """Save screenshot of the agent's current state"""
+    try:
+        # Check if this is an action step with code output
+        if hasattr(step, 'action') and step.action is not None:
+            if hasattr(step.action, 'output'):
+                # Create screenshots directory if it doesn't exist
+                os.makedirs('screenshots', exist_ok=True)
+
+                # Generate filename with timestamp
+                timestamp = datetime.now().strftime("%Y%m%d_%H%M%S_%f")
+                filename = f"screenshots/agent_step_{timestamp}.txt"
+
+                # Save the step information
+                with open(filename, 'w', encoding='utf-8') as f:
+                    f.write(f"Step Type: {type(step).__name__}\n")
+                    f.write(f"Timestamp: {datetime.now().isoformat()}\n")
+                    f.write("=" * 50 + "\n")
+
+                    # Write action details
+                    if hasattr(step.action, 'tool_name'):
+                        f.write(f"Tool: {step.action.tool_name}\n")
+                    if hasattr(step.action, 'arguments'):
+                        f.write(f"Arguments: {step.action.arguments}\n")
+
+                    f.write("\nOutput:\n")
+                    f.write(str(step.action.output))
+
+                print(f"Saved screenshot: {filename}")
+
+    except Exception as e:
+        print(f"Error saving screenshot: {e}")
+
 
 def initialize_driver():
     """Initialize the Selenium WebDriver."""
@@ -159,6 +165,7 @@ Proceed in several steps rather than trying to solve the task in one shot.
 And at the end, only when you have your answer, return your final answer.
 Code:
 ```py
+Make the final answer format is only a string
 final_answer("YOUR_ANSWER_HERE")
 ```<end_code>
 If pages seem stuck on loading, you might have to wait, for instance `import time` and run `time.sleep(5.0)`. But don't overuse this!
@@ -170,11 +177,18 @@ Don't kill the browser.
 When you have modals or cookie banners on screen, you should get rid of them before you can click anything else.
 """
 
+def initialize_agent(model):
+    """Initialize the CodeAgent with the specified model."""
+    return CodeAgent(
+        tools=[DuckDuckGoSearchTool(), go_back, close_popups, search_item_ctrl_f],
+        model=model,
+        additional_authorized_imports=["helium"],
+        step_callbacks=[save_screenshot],
+        max_steps=20,
+        verbosity_level=2,
+    )
 
 def main():
-    # Load environment variables
-    # For example to use an OpenAI model, create a local .env file with OPENAI_API_KEY="<your_open_ai_key_here>"
-    load_dotenv()
 
     # Parse command line arguments
     args = parse_arguments()
@@ -184,13 +198,12 @@ def main():
 
     global driver
     driver = initialize_driver()
-    dynamic_image_agent = initialize_agent(model)
-    dynamic_image_agent.run(agent_request)
+    agent = initialize_agent(model)
 
     # Run the agent with the provided prompt
-    dynamic_image_agent.python_executor("from helium import *")
-    dynamic_image_agent.run(args.prompt + helium_instructions)
+    agent.python_executor("from helium import *")
+    agent.run(args.prompt + helium_instructions)
 
 
 if __name__ == "__main__":
-    main()
+    main()

find_batman_mobile_agent.py

@@ -1,4 +1,7 @@
+import os
 import math
+from PIL import Image
+from io import BytesIO
 from typing import Optional, Tuple
 
 from smolagents import tool
@@ -28,8 +31,8 @@ def calculate_cargo_travel_time(
         return degrees * (math.pi / 180.0)
 
     #extract coordinates
-    lat1, lon1 = map(origin_coords, to_radians)
-    lat2, lon2 = map(destination_coords, to_radians)
+    lat1, lon1 = map(to_radians, origin_coords)
+    lat2, lon2 = map(to_radians, destination_coords)
 
     # Earth Radius in kilometers
     EARTH_RADIUS_KM = 6371.0
@@ -55,7 +58,9 @@ def calculate_cargo_travel_time(
     # Format the results
     return round(flight_time, 2)
 
-print(calculate_cargo_travel_time(41.8781, -87.6298), (-33.8688, 151.2093))
+print(
+    calculate_cargo_travel_time(origin_coords=(41.8781, -87.6298), destination_coords=(-33.8688, 151.2093))
+)
 
 
 def check_reasoning_and_plot(final_answer, agent_memory):

image_agent.py

@@ -1,37 +1,41 @@
-from PTL import find_batman_mobile_agent
+import os
+import PIL
 import requests
 from io import BytesIO
+from PIL import Image
+from smolagents import CodeAgent, OpenAIServerModel, InferenceClientModel
+from huggingface_hub import login
 
 image_urls = [
-    "https://upload.wikimedia.org/wikipedia/commons/e/e8/The_Joker_at_Wax_Museum_Plus.jpg",  # Joker image
-    "https://upload.wikimedia.org/wikipedia/en/9/98/Joker_%28DC_Comics_character%29.jpg"  # Joker image
+    "https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcTcB9-ieIzm2uhtTkMkV5F50xbL4itvh3j8bg&s",  # Joker image
+    "https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcR0SWQC74KuQWrJA33pJqUmOuDx62K7p1uEcQ&s"  # Joker image
 ]
 
-images = []
+stored_images = []
 
 for url in image_urls:
     headers = {
         "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"
     }
     response = requests.get(url, headers)
-    image = Image.open(BytesIO(response.content)).convert("RGB")
-    images.append(image)
+    try:
+        image = Image.open(BytesIO(response.content)).convert("RGB")
+        stored_images.append(image)
+    except PIL.UnidentifiedImageError:
+        print("⚠️ The downloaded file is not a valid image.")
+        # Optional: save the raw bytes for inspection
+        with open("debug_image.bin", "wb") as f:
+            f.write(response.content)
+    # image = Image.open(BytesIO(response.content)).convert("RGB")
 
 
-# Now based on the images indicate if it's wonder women or joker.
-model = OpenAIServerModel(model="gpt-4o")
-agent = CodeAgent(
-    tools=[],
-    model=model,
-    max_steps = 20,
-    verbosity_level=2
-)
-
-response = agent.run(
-    """
-    Describe the costume and makeup that the comic characteer in these photos is wearing and return the description.
-    Tell me if the guest is the Joker or Wonder Women
-    """,
-    images = images
-)
-
+def init_stored_image_agent() -> CodeAgent:
+    # Now based on the images indicate if it's wonder women or joker.
+    model = OpenAIServerModel(model_id="gpt-4o")
+    stored_image_agent = CodeAgent(
+        tools=[],
+        model=model,
+        max_steps=5,
+        verbosity_level=2
+    )
+    return stored_image_agent

mulitagents.py

@@ -4,80 +4,66 @@ from smolagents import CodeAgent, DuckDuckGoSearchTool, InferenceClientModel, Vi
 
 from find_batman_mobile_agent import calculate_cargo_travel_time, check_reasoning_and_plot
 
-example_model = InferenceClientModel(model_id="Qwen/Qwen2.5-Coder-32B-Instruct", provider="together")
 
-task = """Find all Batman filming locations in the world, calculate the time to transfer via cargo plane to here (we're in Gotham, 40.7128, 74.0060), and return them to me as a pandas dataframe.
-Also give me some supercar factories with the same cargo plane transfer time.
-"""
 
-example_multi_agent = CodeAgent(
-    model=example_model,
-    tools=[DuckDuckGoSearchTool(), VisitWebpageTool(), calculate_cargo_travel_time()],
-    additional_authorized_imports=["pandas"],
-    max_steps=20
-)
+def define_multi_agent():
+    example_model = InferenceClientModel(model_id="Qwen/Qwen2.5-Coder-32B-Instruct", provider="together")
+
+    task = """Find all Batman filming locations in the world, calculate the time to transfer via cargo plane to here (we're in Gotham, 40.7128, 74.0060), and return them to me as a pandas dataframe.
+    Also give me some supercar factories with the same cargo plane transfer time.
+    """
+
+    example_multi_agent = CodeAgent(
+        model=example_model,
+        tools=[DuckDuckGoSearchTool(), VisitWebpageTool(), calculate_cargo_travel_time],
+        additional_authorized_imports=["pandas"],
+        max_steps=20
+    )
+
+    result = example_multi_agent.run(task)
+
+    example_multi_agent.planning_interval = 4
+
+    detailed_report = example_multi_agent.run(f"""
+        You're an expert analyst. You make comprehensive reports after visiting many websites.
+        Don't hesitate to search for many queries at once in a for loop.
+        For each data point that you find, visit the source url to confirm numbers.
+    """)
+
+    print(detailed_report)
+
+    example_web_model = InferenceClientModel(
+        "Qwen/Qwen2.5-Coder-32B-Instruct", provider="together", max_tokens=8096
+    )
+
+    example_web_agent = CodeAgent(
+        model=example_web_model,
+        tools=[
+            DuckDuckGoSearchTool(),
+            VisitWebpageTool(),
+            calculate_cargo_travel_time,
+        ],
+        name="web_agent",
+        description="Browses the web to find information",
+        verbosity_level=0,
+        max_steps=10
+    )
+    manager_agent = CodeAgent(
+        model=InferenceClientModel("deepseek-ai/DeepSeek-R1", provider="together", max_tokens=8096),
+        tools=[calculate_cargo_travel_time],
+        managed_agents=[example_web_agent],
+        additional_authorized_imports=[
+            "geopandas",
+            "plotly",
+            "shapely",
+            "json",
+            "pandas",
+            "numpy"
+        ],
+        planning_interval=5,
+        verbosity_level=2,
+        final_answer_checks=[check_reasoning_and_plot],
+        max_steps=15
+    )
+    return manager_agent
 
-result = example_multi_agent.run(task)
-
-example_multi_agent.planning_interval = 4
-
-detailed_report = example_multi_agent.run(f"""
-    You're an expert analyst. You make comprehensive reports after visiting many websites.
-    Don't hesitate to search for many queries at once in a for loop.
-    For each data point that you find, visit the source url to confirm numbers.
-""")
-
-print(detailed_report)
-
-example_web_model = InferenceClientModel(
-    "Qwen/Qwen2.5-Coder-32B-Instruct", provider="together", max_tokens=8096
-)
-
-example_web_agent = CodeAgent(
-    model=example_web_model,
-    tools=[
-        DuckDuckGoSearchTool(),
-        VisitWebpageTool(),
-        calculate_cargo_travel_time(),
-    ],
-    name = "web_agent",
-    description="Browses the web to find information",
-    verbosity_level=0,
-    max_steps=10
-)
-
-manager_agent = CodeAgent(
-    model=InferenceClientModel("deepseek-ai/DeepSeek-R1", provider="together", max_tokens=8096),
-    tools=[calculate_cargo_travel_time],
-    managed_agents=[example_web_agent],
-    additional_authorized_imports=[
-        "geopandas",
-        "plotly",
-        "shapely",
-        "json",
-        "pandas",
-        "numpy"
-    ],
-    planning_interval=5,
-    verbosity_level=2,
-    final_answer_checks=[check_reasoning_and_plot],
-    max_steps=15
-)
-
-manager_agent.visualize()
-
-manager_agent.run("""
-Find all Batman filming locations in the world, calculate the time to transfer via cargo plane to here (we're in Gotham, 40.7128° N, 74.0060° W).
-Also give me some supercar factories with the same cargo plane transfer time. You need at least 6 points in total.
-Represent this as spatial map of the world, with the locations represented as scatter points with a color that depends on the travel time, and save it to saved_map.png!
-
-Here's an example of how to plot and return a map:
-import plotly.express as px
-df = px.data.carshare()
-fig = px.scatter_map(df, lat="centroid_lat", lon="centroid_lon", text="name", color="peak_hour", size=100
-color_continous_scale=px.colors.sequential.Magma, size_max=15, zoom=1)
-
-Never try to process strings using code: when you have a string to read, just print it and you'll see it.
-""")
-
-manager_agent.python_executor.state["fig"]

nsfw_detection.py

@@ -0,0 +1,63 @@
+from PIL import Image
+import requests
+from io import BytesIO
+import base64
+import re
+from transformers import pipeline
+
+classifier = pipeline("image-classification", model="Falconsai/nsfw_image_detection")
+
+
+def classify_image_if_nsfw(image_url: str):
+    try:
+        # Check if it's a base64 data URL
+        if image_url.startswith('data:image'):
+            print("Processing base64 data URL")
+
+            # Extract the base64 data from the data URL
+            match = re.match(r'data:image/(?P<ext>\w+);base64,(?P<data>.*)', image_url)
+            if not match:
+                raise ValueError("Invalid base64 data URL format")
+
+            base64_data = match.group('data')
+            image_format = match.group('ext')
+
+            # Decode the base64 data
+            image_data = base64.b64decode(base64_data)
+
+            # Open the image from decoded data
+            img = Image.open(BytesIO(image_data))
+
+        else:
+            # It's a regular URL - download the image
+            print("Processing regular URL")
+            response = requests.get(image_url)
+            response.raise_for_status()
+
+            # Open and process the image
+            img = Image.open(BytesIO(response.content))
+
+        print("Image size:", img.size)
+        print("Image format:", img.format)
+        print("Image mode:", img.mode)
+
+        # Ensure image is in RGB mode (required by most models)
+        if img.mode != 'RGB':
+            img = img.convert('RGB')
+
+        # Classify the image
+        classifier_response = classifier(img)
+        print("Classifier Response:", classifier_response)
+
+        return classifier_response
+
+    except Exception as e:
+        print(f"Error processing image: {e}")
+        raise
+
+# Example usage with both types:
+# Regular URL
+# result1 = classify_image_if_nsfw("https://example.com/image.jpg")
+
+# Base64 data URL (you would use an actual base64 string here)
+# result2 = classify_image_if_nsfw("data:image/jpeg;base64,/9j/4AAQSkZJRgABAQ...")

requirements.txt

@@ -1,8 +1,15 @@
-gradio
-huggingface_hub
-smolagents[all]
-requests
-pillow
-selenium
-helium
-langchain-community
+gradio~=5.49.1
+huggingface_hub~=1.0.1
+smolagents[all]~=1.22.0
+requests~=2.32.4
+pillow~=11.3.0
+selenium~=4.38.0
+helium~=5.1.1
+langchain-community~=0.3.27
+litellm
+transformers~=4.53.2
+python-dotenv~=1.1.1
+plotly
+shapely
+geopandas
+markdownify