Added Transformers models

agents/all_agents.py

@@ -80,7 +80,6 @@ def get_master_agent(llm):
             ImageResizeTool(),
             # UpscalerTool(),
         ],
-        use_structured_outputs_internally=True,
         verbosity_level=LogLevel.DEBUG,
     )
     print("Loaded master agent")

app.py

@@ -7,7 +7,7 @@ from gradio import ChatMessage
 from smolagents.gradio_ui import stream_to_gradio
 
 from agents.all_agents import get_master_agent
-from llm import ANTHROPIC_MODEL_IDS, get_anthropic_model
+from llm import TRANSFORMERS_MODEL_IDS, get_transformers_model
 
 
 def resize_image(image):
@@ -22,8 +22,8 @@ def resize_image(image):
 
 
 @spaces.GPU
-def chat_interface_fn(input_request, history: List[ChatMessage], gallery, anthropic_api_key, anthropic_model_id):
-    model = get_anthropic_model(anthropic_model_id, anthropic_api_key)
+def chat_interface_fn(input_request, history: List[ChatMessage], gallery, transformers_model_id):
+    model = get_transformers_model(transformers_model_id)
     agent = get_master_agent(model)
     if gallery is None:
         gallery = []
@@ -126,22 +126,13 @@ with gr.Blocks() as demo:
     gr.Markdown(
         """
         ## Update 17/06/2025
-        This Space was originally a Hackathon submission, funded with Anthropic Free Credits.<br>
-        Due to the high popularity of the Space, unfortunately I can't fund personally the credits anymore.<br>
-        I have added below the ability to add your own Anthropic API Key and select the model to use.<br>
-        """
-    )
-    anthropic_api_key = gr.Textbox(label="Anthropic API Key")
-    anthropic_model_id = gr.Dropdown(label="Anthropic Model", choices=ANTHROPIC_MODEL_IDS)
-    gr.Markdown(
-        """
-        ## Future plans
-        I plan to continue developing this Space on a more personal space here : https://huggingface.co/spaces/stevenbucaille/ScouterAI <br>
-        This Space will be powered with ZeroGPU and have more LLM options.<br>
-        Stay tuned!
+        Welcome to the new version of ScouterAI!<br>
+        This Space is now powered by ZeroGPU, which means you can use it for free!<br>
+        You can now use any model from the HuggingFace Hub, just provide the model ID below.<br>
         <br>
         """
     )
+    transformers_model_id = gr.Dropdown(label="Transformers Model ID", info="Select a model to use or provide your own model ID", choices=TRANSFORMERS_MODEL_IDS)
     output_gallery = gr.Gallery(label="Images generated by the agent (do not put images)", type="pil", format="png")
     textbox = gr.MultimodalTextbox()
     gr.ChatInterface(
@@ -149,7 +140,7 @@ with gr.Blocks() as demo:
         type="messages",
         multimodal=True,
         textbox=textbox,
-        additional_inputs=[output_gallery, anthropic_api_key, anthropic_model_id],
+        additional_inputs=[output_gallery, transformers_model_id],
         additional_outputs=[output_gallery],
     )
 

llm.py

@@ -1,4 +1,4 @@
-from smolagents import LiteLLMModel
+from smolagents import LiteLLMModel, TransformersModel
 
 
 ANTHROPIC_MODEL_IDS = [
@@ -13,6 +13,14 @@ ANTHROPIC_MODEL_IDS = [
     "claude-3-haiku-20240307",
 ]
 
+TRANSFORMERS_MODEL_IDS = [
+    "Qwen/Qwen2.5-VL-3B-Instruct",
+    "Qwen/Qwen2.5-VL-7B-Instruct",
+    "Qwen/Qwen2.5-VL-14B-Instruct",
+    "Qwen/Qwen2.5-VL-32B-Instruct",
+    "Qwen/Qwen2.5-VL-72B-Instruct",
+]
+
 
 def get_anthropic_model(model_id, anthropic_api_key):
     if model_id not in ANTHROPIC_MODEL_IDS:
@@ -22,3 +30,8 @@ def get_anthropic_model(model_id, anthropic_api_key):
         api_key=anthropic_api_key,
     )
     return model
+
+
+def get_transformers_model(model_id):
+    model = TransformersModel(model_id=model_id)
+    return model

pyproject.toml

@@ -12,7 +12,7 @@ dependencies = [
     "faiss-gpu>=1.7.2",
     "gradio>=5.33.0",
     "hf-transfer>=0.1.9",
-    "huggingface-hub[cli]>=0.32.4",
+    "huggingface-hub[cli,hf-xet]>=0.32.4",
     "langchain>=0.3.25",
     "langchain-community>=0.3.24",
     "langchain-huggingface>=0.2.0",
@@ -25,7 +25,8 @@ dependencies = [
     "safetensors>=0.5.3",
     "scipy>=1.15.3",
     "sentence-transformers>=4.1.0",
-    "smolagents[litellm,mcp,openai]>=1.17.0",
+    "smolagents[litellm,mcp,openai,vllm]>=1.17.0",
+    "spaces>=0.37.0",
     "supervision>=0.25.1",
     "timm>=1.0.15",
     "torch==2.5.1",

requirements.txt

@@ -5,30 +5,44 @@ aiofiles==24.1.0
 aiohappyeyeballs==2.6.1
 aiohttp==3.12.9
 aiosignal==1.3.2
+airportsdata==20250523
 annotated-types==0.7.0
 anyio==4.9.0
+astor==0.8.1
 async-timeout==4.0.3
 attrs==25.3.0
+blake3==1.0.5
 certifi==2025.4.26
 charset-normalizer==3.4.2
 click==8.1.8
+cloudpickle==3.1.1
+compressed-tensors==0.9.1
 contourpy==1.3.2
+cupy-cuda12x==13.4.1
 cycler==0.12.1
 dataclasses-json==0.6.7
 datasets==3.6.0
 defusedxml==0.7.1
+depyf==0.18.0
 diffusers==0.33.1
 dill==0.3.8
+diskcache==5.6.3
 distro==1.9.0
+dnspython==2.7.0
+einops==0.8.1
+email-validator==2.2.0
 exceptiongroup==1.3.0
 faiss-cpu==1.11.0
 faiss-gpu==1.7.2
 fastapi==0.115.12
+fastapi-cli==0.0.7
+fastrlock==0.8.3
 ffmpy==0.6.0
 filelock==3.18.0
 fonttools==4.58.1
 frozenlist==1.6.2
 fsspec==2025.3.0
+gguf==0.10.0
 gradio==5.33.0
 gradio-client==1.10.2
 greenlet==3.2.3
@@ -40,13 +54,16 @@ hf-transfer==0.1.9
 hf-xet==1.1.3
 hpack==4.1.0
 httpcore==1.0.9
+httptools==0.6.4
 httpx==0.28.1
 httpx-sse==0.4.0
 huggingface-hub==0.32.4
 hyperframe==6.1.0
 idna==3.10
 importlib-metadata==8.7.0
+iniconfig==2.1.0
 inquirerpy==0.3.4
+interegular==0.3.3
 jinja2==3.1.6
 jiter==0.10.0
 joblib==1.5.1
@@ -63,7 +80,10 @@ langchain-huggingface==0.2.0
 langchain-openai==0.3.21
 langchain-text-splitters==0.3.8
 langsmith==0.3.45
+lark==1.2.2
 litellm==1.72.1
+llvmlite==0.43.0
+lm-format-enforcer==0.10.11
 markdown-it-py==3.0.0
 markupsafe==3.0.2
 marshmallow==3.26.1
@@ -71,13 +91,18 @@ matplotlib==3.10.3
 mcp==1.9.3
 mcpadapt==0.1.9
 mdurl==0.1.2
+mistral-common==1.6.2
 modal==1.0.3
 mpmath==1.3.0
+msgpack==1.1.1
+msgspec==0.19.0
 multidict==6.4.4
 multiprocess==0.70.16
 mypy-extensions==1.1.0
+nest-asyncio==1.6.0
 networkx==3.4.2
-numpy==2.2.6
+numba==0.60.0
+numpy==1.26.4
 nvidia-cublas-cu12==12.4.5.8
 nvidia-cuda-cupti-cu12==12.4.127
 nvidia-cuda-nvrtc-cu12==12.4.127
@@ -92,33 +117,47 @@ nvidia-nvjitlink-cu12==12.4.127
 nvidia-nvtx-cu12==12.4.127
 openai==1.84.0
 opencv-python==4.11.0.86
+opencv-python-headless==4.11.0.86
 orjson==3.10.18
+outlines==0.1.11
+outlines-core==0.1.26
 packaging==24.2
 pandas==2.3.0
+partial-json-parser==0.2.1.1.post5
 pfzy==0.3.4
 pillow==11.2.1
+pluggy==1.6.0
+prometheus-client==0.22.1
+prometheus-fastapi-instrumentator==7.1.0
 prompt-toolkit==3.0.51
 propcache==0.3.1
 protobuf==6.31.1
 psutil==5.9.8
+py-cpuinfo==9.0.0
 pyarrow==20.0.0
+pybind11==2.13.6
+pycountry==24.6.1
 pydantic==2.11.5
 pydantic-core==2.33.2
 pydantic-settings==2.9.1
 pydub==0.25.1
 pygments==2.19.1
 pyparsing==3.2.3
+pytest==8.4.0
 python-dateutil==2.9.0.post0
 python-dotenv==1.1.0
 python-multipart==0.0.20
 pytz==2025.2
 pyyaml==6.0.2
+pyzmq==27.0.0
 rank-bm25==0.2.2
+ray==2.40.0
 referencing==0.36.2
 regex==2024.11.6
 requests==2.32.3
 requests-toolbelt==1.0.0
 rich==14.0.0
+rich-toolkit==0.14.7
 rpds-py==0.25.1
 ruff==0.11.12
 safehttpx==0.1.6
@@ -127,6 +166,7 @@ scikit-learn==1.7.0
 scipy==1.15.3
 semantic-version==2.10.0
 sentence-transformers==4.1.0
+sentencepiece==0.2.0
 shellingham==1.5.4
 sigtools==4.0.1
 six==1.17.0
@@ -145,8 +185,10 @@ tiktoken==0.9.0
 timm==1.0.15
 tokenizers==0.21.1
 toml==0.10.2
+tomli==2.2.1
 tomlkit==0.13.3
 torch==2.5.1
+torchaudio==2.5.1
 torchvision==0.20.1
 tqdm==4.67.1
 transformers==4.52.4
@@ -160,9 +202,13 @@ typing-inspection==0.4.1
 tzdata==2025.2
 urllib3==2.4.0
 uvicorn==0.34.3
+uvloop==0.21.0
+vllm==0.7.3
 watchfiles==1.0.5
 wcwidth==0.2.13
 websockets==15.0.1
+xformers==0.0.28.post3
+xgrammar==0.1.11
 xxhash==3.5.0
 yarl==1.20.0
 zipp==3.22.0