fix vision

agent.py

@@ -1,4 +1,8 @@
 import os
+import base64
+import requests
+import json
+import traceback
 import datetime
 import subprocess
 import tempfile
@@ -16,9 +20,7 @@ from groq import Groq
 from langchain_groq import ChatGroq
 from langchain_community.document_loaders.image import UnstructuredImageLoader
 from langchain_community.document_loaders import WebBaseLoader
-from langchain_openai import ChatOpenAI
 from langchain_google_genai import ChatGoogleGenerativeAI
-import base64
 
 try:
     import cv2
@@ -36,7 +38,7 @@ def get_whisper():
         whisper_model = whisper.load_model("base")
     return whisper_model
 
-load_dotenv()
+load_dotenv(override=True)
 
 # Base Hugging Face LLM used by the chat wrapper
 # base_llm = HuggingFaceEndpoint(
@@ -55,14 +57,14 @@ def smart_invoke(msgs, use_tools=False, start_tier=0):
     Retries next tier if a 429 (rate limit), 402 (credits), or 404 (model found) error occurs.
     """
     
-    # Adaptive Gemini names to try if 1.5 flash is 404
-    gemini_alternatives = ["gemini-2.5-flash-lite", "gemma-3-1b", "gemini-3-flash", "gemini-3.1-flash-lite"]
+    # Adaptive Gemini names to try if 3.1 flash is 404
+    gemini_alternatives = ["gemini-2.0-flash", "gemini-3.1-flash-lite", "gemini-3.1-pro"]
     
     tiers_config = [
         {"name": "OpenRouter", "key": "OPENROUTER_API_KEY", "provider": "openai", "model_name": "meta-llama/llama-3.3-70b-instruct", "base_url": "https://openrouter.ai/api/v1"},
-        {"name": "Gemini", "key": "GOOGLE_API_KEY", "provider": "google", "model_name": "gemini-2.5-flash", "alternatives": gemini_alternatives},
-        {"name": "Groq", "key": "GROQ_API_KEY", "provider": "groq", "model_name": "meta-llama/llama-4-scout-17b-16e-instruct"},
-        {"name": "NVIDIA", "key": "NVIDIA_API_KEY", "provider": "openai", "model_name": "meta/llama-3.1-405b-instruct", "base_url": "https://integrate.api.nvidia.com/v1"},
+        {"name": "Gemini", "key": "GOOGLE_API_KEY", "provider": "google", "model_name": "gemini-2.0-flash", "alternatives": gemini_alternatives},
+        {"name": "Groq", "key": "GROQ_API_KEY", "provider": "groq", "model_name": "llama-3.3-70b-versatile"},
+        {"name": "NVIDIA", "key": "NVIDIA_API_KEY", "provider": "openai", "model_name": "meta/llama-3.3-70b-instruct", "base_url": "https://integrate.api.nvidia.com/v1"},
         {"name": "Vercel", "key": "VERCEL_API_KEY", "provider": "openai", "model_name": "meta-llama/llama-3.3-70b-instruct", "base_url": "https://gateway.ai.vercel.com/v1"},
     ]
     
@@ -75,38 +77,13 @@ def smart_invoke(msgs, use_tools=False, start_tier=0):
             
         def create_model_instance(m_name, provider, b_url=None):
             if provider == "openai":
+                from langchain_openai import ChatOpenAI
                 return ChatOpenAI(model=m_name, openai_api_key=api_key, openai_api_base=b_url, temperature=0)
             elif provider == "google":
+                from langchain_google_genai import ChatGoogleGenerativeAI
                 return ChatGoogleGenerativeAI(model=m_name, temperature=0)
             elif provider == "groq":
-                return ChatGroq(model=m_name, temperature=0, max_retries=2)
-            return None
-
-        primary_model = create_model_instance(tier["model_name"], tier["provider"], tier.get("base_url"))
-        if use_tools:
-            primary_model = primary_model.bind_tools(tools)
-            
-        models_to_try = [primary_model]
-        if "alternatives" in tier:
-            for alt_name in tier["alternatives"]:
-                alt_model = create_model_instance(alt_name, tier["provider"], tier.get("base_url"))
-                if use_tools:
-                    alt_model = alt_model.bind_tools(tools)
-                models_to_try.append(alt_model)
-    
-    last_exception = None
-    for i in range(start_tier, len(tiers_config)):
-        tier = tiers_config[i]
-        api_key = os.getenv(tier["key"])
-        if not api_key:
-            continue 
-            
-        def create_model_instance(m_name, provider, b_url=None):
-            if provider == "openai":
-                return ChatOpenAI(model=m_name, openai_api_key=api_key, openai_api_base=b_url, temperature=0)
-            elif provider == "google":
-                return ChatGoogleGenerativeAI(model=m_name, temperature=0)
-            elif provider == "groq":
+                from langchain_groq import ChatGroq
                 return ChatGroq(model=m_name, temperature=0, max_retries=2)
             return None
 
@@ -135,7 +112,7 @@ def smart_invoke(msgs, use_tools=False, start_tier=0):
                     continue
                 
                 # Catch other fallback triggers
-                if any(x in err_str for x in ["rate_limit", "429", "500", "503", "overloaded", "not_found", "404", "402", "credits"]):
+                if any(x in err_str for x in ["rate_limit", "429", "500", "503", "overloaded", "not_found", "404", "402", "credits", "decommissioned", "invalid_request_error"]):
                     print(f"--- {tier['name']} Error: {e}. Trying next model/tier... ---")
                     last_exception = e
                     # If this tier has more alternatives, continue to the next one
@@ -198,7 +175,32 @@ def wiki_search(query: str) -> str:
         ])
     return formatted_search_docs
 
-    
+def get_vision_models():
+    """Returns a list of vision models to try, in order of preference."""
+    configs = [
+        {"name": "OpenRouter-Gemini-2.0", "key": "OPENROUTER_API_KEY", "provider": "openai", "model_name": "google/gemini-2.0-flash-001", "base_url": "https://openrouter.ai/api/v1"},
+        {"name": "Google-Gemini-2.0-Exp", "key": "GOOGLE_API_KEY", "provider": "google", "model_name": "gemini-2.0-flash-exp"},
+        {"name": "Google-Gemini-1.5-Latest", "key": "GOOGLE_API_KEY", "provider": "google", "model_name": "gemini-1.5-flash-latest"},
+        {"name": "NVIDIA-Vision-Llama-11b", "key": "NVIDIA_API_KEY", "provider": "openai", "model_name": "meta/llama-3.2-11b-vision-instruct", "base_url": "https://integrate.api.nvidia.com/v1"},
+        {"name": "NVIDIA-Vision-Llama-90b", "key": "NVIDIA_API_KEY", "provider": "openai", "model_name": "meta/llama-3.2-90b-vision-instruct", "base_url": "https://integrate.api.nvidia.com/v1"},
+        {"name": "Groq-Vision", "key": "GROQ_API_KEY", "provider": "groq", "model_name": "llama-3.2-90b-vision-preview"},
+    ]
+    models = []
+    for cfg in configs:
+        api_key = os.getenv(cfg["key"])
+        if not api_key:
+            continue
+        if cfg["provider"] == "openai":
+            from langchain_openai import ChatOpenAI
+            m = ChatOpenAI(model=cfg["model_name"], openai_api_key=api_key, openai_api_base=cfg.get("base_url"), temperature=0)
+        elif cfg["provider"] == "google":
+            from langchain_google_genai import ChatGoogleGenerativeAI
+            m = ChatGoogleGenerativeAI(model=cfg["model_name"], temperature=0)
+        elif cfg["provider"] == "groq":
+            from langchain_groq import ChatGroq
+            m = ChatGroq(model=cfg["model_name"], temperature=0)
+        models.append({"name": cfg["name"], "model": m})
+    return models
 
 @tool
 def analyze_image(image_path: str, question: str) -> str:
@@ -212,18 +214,13 @@ def analyze_image(image_path: str, question: str) -> str:
         question: Specific question describing what you want the vision model to look for.
     """
     try:
+        if not os.path.exists(image_path):
+            return f"Error: Image file not found at {image_path}"
+            
         # If it's a local file, we encode it to base64
         with open(image_path, "rb") as image_file:
             encoded_image = base64.b64encode(image_file.read()).decode('utf-8')
             
-        # Use OpenRouter for Vision as a more robust fallback
-        vision_model = ChatOpenAI(
-            model="google/gemini-2.0-flash-001",
-            openai_api_key=os.getenv("OPENROUTER_API_KEY"),
-            openai_api_base="https://openrouter.ai/api/v1",
-            temperature=0,
-        )
-        
         message = HumanMessage(
             content=[
                 {"type": "text", "text": question},
@@ -233,10 +230,26 @@ def analyze_image(image_path: str, question: str) -> str:
                 },
             ]
         )
-        response = vision_model.invoke([message])
-        return response.content
+        
+        vision_models = get_vision_models()
+        if not vision_models:
+            return "Error: No vision models configured (missing API keys)."
+            
+        last_err = None
+        for item in vision_models:
+            try:
+                m_name = getattr(item['model'], 'model', 'unknown')
+                print(f"--- Calling Vision Model: {item['name']} ({m_name}) ---")
+                response = item['model'].invoke([message])
+                return extract_text_from_content(response.content)
+            except Exception as e:
+                print(f"Vision Model {item['name']} failed.")
+                traceback.print_exc()
+                last_err = e
+        return f"Error analyzing image: All vision models failed. Last error: {str(last_err)}"
     except Exception as e:
-        return f"Error analyzing image: {str(e)}"
+        traceback.print_exc()
+        return f"Error reading/processing image: {str(e)}"
 
 @tool
 def analyze_audio(audio_path: str, question: str) -> str:
@@ -279,7 +292,8 @@ def analyze_video(video_path: str, question: str) -> str:
         frame_indices = [int(i * total_frames / 5) for i in range(5)]
         extracted_descriptions = []
         
-        vision_model = ChatGroq(model="llama-3.2-90b-vision-preview", temperature=0)
+        vision_models = get_vision_models()
+        # Ensure Groq-Llama is at the front for video if preferred, but we'll use the default order for now.
         
         for idx_num, frame_idx in enumerate(frame_indices):
             cap.set(cv2.CAP_PROP_POS_FRAMES, frame_idx)
@@ -289,14 +303,24 @@ def analyze_video(video_path: str, question: str) -> str:
                 _, buffer = cv2.imencode('.jpg', frame)
                 encoded_image = base64.b64encode(buffer).decode('utf-8')
                 
-                # Ask the vision model to describe the frame
+                # Ask a vision model to describe the frame (with fallback)
                 msg = HumanMessage(
                     content=[
                         {"type": "text", "text": f"Describe what is happening in this video frame concisely. Focus on aspects related to: {question}"},
                         {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{encoded_image}"}},
                     ]
                 )
-                desc = vision_model.invoke([msg]).content
+                
+                desc = "No description available."
+                for item in vision_models:
+                    try:
+                        print(f"--- Calling Vision Model for Frame {idx_num+1}: {item['name']} ---")
+                        desc = item['model'].invoke([msg]).content
+                        break
+                    except Exception as e:
+                        print(f"Vision Model {item['name']} failed for frame: {e}")
+                        continue
+                
                 extracted_descriptions.append(f"Frame {idx_num + 1}: {desc}")
                 
         cap.release()
@@ -424,7 +448,21 @@ def restart_required(state: AgentState) -> AgentState:
 # Augment the LLM with tools
 tools = [web_search, wiki_search, analyze_image, analyze_audio, analyze_video, read_url, run_python_script, read_document]
 tools_by_name = {tool.name: tool for tool in tools}
-# model_with_tools etc. removed, replaced by lazy initialization in smart_invoke
+def extract_text_from_content(content: Any) -> str:
+    """Extracts a simple string from various possible AIMessage content formats."""
+    if isinstance(content, str):
+        return content
+    if isinstance(content, list):
+        text_parts = []
+        for part in content:
+            if isinstance(part, str):
+                text_parts.append(part)
+            elif isinstance(part, dict) and "text" in part:
+                text_parts.append(part["text"])
+            elif isinstance(part, dict) and "type" in part and part["type"] == "text":
+                text_parts.append(part.get("text", ""))
+        return "".join(text_parts)
+    return str(content)
 
 def answer_message(state: AgentState) -> AgentState:
     messages = state["messages"]
@@ -503,7 +541,7 @@ def answer_message(state: AgentState) -> AgentState:
         print("Max reasoning steps reached. Forcing answer extraction.")
         forced_msg = HumanMessage(content="You have reached the maximum reasoning steps. Please provide your best final answer based on the current context without any more tool calls.")
         messages.append(forced_msg)
-        draft_response = smart_invoke(messages, use_tools=False)
+        draft_response, _ = smart_invoke(messages, use_tools=False)
 
     # Third pass: strict GAIA formatting extraction
     formatting_sys = SystemMessage(
@@ -516,11 +554,15 @@ def answer_message(state: AgentState) -> AgentState:
             "If it is a name or word, just return the exact string. If a list, return only the comma-separated list."
         )
     )
-    final_response, _ = smart_invoke([formatting_sys, HumanMessage(content=draft_response.content)], use_tools=False, start_tier=current_tier)
+    final_response, _ = smart_invoke([formatting_sys, HumanMessage(content=extract_text_from_content(draft_response.content))], use_tools=False, start_tier=current_tier)
     print(f"Draft response: {draft_response.content}")
     print(f"Strict Final response: {final_response.content}")
 
     # Return messages including the final AIMessage so BasicAgent reads .content
+    # Ensure final_response has string content for basic agents
+    if not isinstance(final_response.content, str):
+        final_response.content = extract_text_from_content(final_response.content)
+        
     messages.append(draft_response)
     messages.append(final_response)
     return {"messages": messages}

check_env.py

@@ -0,0 +1,21 @@
+import os
+from dotenv import load_dotenv
+
+# Try to load .env from current directory
+env_path = os.path.join(os.getcwd(), '.env')
+print(f"Checking for .env at: {env_path}")
+print(f"File exists: {os.path.exists(env_path)}")
+
+load_dotenv(env_path)
+
+# Print keys (masking values)
+keys = list(os.environ.keys())
+relevant_keys = [k for k in keys if any(x in k for x in ["API_KEY", "TOKEN", "GOOGLE", "GROQ", "NVIDIA", "VERCEL", "OPENROUTER"])]
+print(f"Relevant keys found: {relevant_keys}")
+
+# Specifically check the ones we need
+needed = ["NVIDIA_API_KEY", "VERCEL_API_KEY", "OPENROUTER_API_KEY", "GOOGLE_API_KEY", "GROQ_API_KEY"]
+for k in needed:
+    val = os.getenv(k)
+    status = "PRESENT (length={})".format(len(val)) if val else "MISSING"
+    print(f"{k}: {status}")

check_env_v2.py

@@ -0,0 +1,25 @@
+import os
+from dotenv import load_dotenv
+
+# Try to load .env from current directory with override=True
+env_path = os.path.join(os.getcwd(), '.env')
+print(f"Checking for .env at: {env_path}")
+print(f"File exists: {os.path.exists(env_path)}")
+
+load_dotenv(env_path, override=True)
+
+# Print keys (case-insensitive check)
+keys = list(os.environ.keys())
+relevant_keys = [k for k in keys if any(x in k.upper() for x in ["API_KEY", "TOKEN", "GOOGLE", "GROQ", "NVIDIA", "VERCEL", "OPENROUTER"])]
+print(f"Relevant keys found: {relevant_keys}")
+
+# Check specifically
+needed = ["NVIDIA_API_KEY", "VERCEL_API_KEY", "OPENROUTER_API_KEY", "GOOGLE_API_KEY", "GROQ_API_KEY"]
+for k in needed:
+    # Try case-insensitive lookup
+    found_key = next((key for key in keys if key.upper() == k), None)
+    if found_key:
+        val = os.getenv(found_key)
+        print(f"{found_key}: PRESENT (length={len(val)})")
+    else:
+        print(f"{k}: MISSING")

test_vision.py

@@ -0,0 +1,35 @@
+import os
+from langchain_openai import ChatOpenAI
+from langchain_core.messages import HumanMessage
+import base64
+from dotenv import load_dotenv
+
+load_dotenv()
+
+def test_vision():
+    # Use a tiny 1x1 base64 image for testing
+    tiny_img = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg=="
+    msg = HumanMessage(content=[{"type": "text", "text": "what is in this image?"}, {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{tiny_img}"}}])
+    
+    models = [
+        {"name": "OpenRouter-Gemini-2.0", "provider": "openai", "model": "google/gemini-2.0-flash-001", "base_url": "https://openrouter.ai/api/v1", "key": "OPENROUTER_API_KEY"},
+        {"name": "NVIDIA-Llama-3.2", "provider": "openai", "model": "nvidia/llama-3.2-nv-vision-70b", "base_url": "https://integrate.api.nvidia.com/v1", "key": "NVIDIA_API_KEY"},
+        {"name": "NVIDIA-Qwen-VL", "provider": "openai", "model": "nvidia/qwen-vl-max", "base_url": "https://integrate.api.nvidia.com/v1", "key": "NVIDIA_API_KEY"},
+        {"name": "Vercel-Vision", "provider": "openai", "model": "gpt-4o-mini", "base_url": "https://gateway.ai.vercel.com/v1", "key": "VERCEL_API_KEY"},
+    ]
+    
+    for m in models:
+        key = os.getenv(m['key'])
+        if not key:
+            print(f"Skip {m['name']} (no key)")
+            continue
+        try:
+            print(f"Testing {m['name']} ({m['model']})...")
+            llm = ChatOpenAI(model=m['model'], openai_api_key=key, openai_api_base=m['base_url'], temperature=0)
+            res = llm.invoke([msg])
+            print(f"Success: {res.content}")
+        except Exception as e:
+            print(f"Fail: {e}")
+
+if __name__ == "__main__":
+    test_vision()

test_vision_v2.py

@@ -0,0 +1,34 @@
+import os
+from langchain_openai import ChatOpenAI
+from langchain_core.messages import HumanMessage
+import base64
+from dotenv import load_dotenv
+
+load_dotenv(override=True)
+
+def test_vision():
+    # Use a tiny 1x1 base64 image for testing
+    tiny_img = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg=="
+    msg = HumanMessage(content=[{"type": "text", "text": "is this image red, green, or blue? answer with one word."}, {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{tiny_img}"}}])
+    
+    models = [
+        {"name": "NVIDIA-Llama-3.2-11b", "provider": "openai", "model": "meta/llama-3.2-11b-vision-instruct", "base_url": "https://integrate.api.nvidia.com/v1", "key": "NVIDIA_API_KEY"},
+        {"name": "NVIDIA-Llama-3.2-90b", "provider": "openai", "model": "meta/llama-3.2-90b-vision-instruct", "base_url": "https://integrate.api.nvidia.com/v1", "key": "NVIDIA_API_KEY"},
+        {"name": "NVIDIA-Mistral-Vision", "provider": "openai", "model": "mistralai/pixtral-12b", "base_url": "https://integrate.api.nvidia.com/v1", "key": "NVIDIA_API_KEY"},
+    ]
+    
+    for m in models:
+        key = os.getenv(m['key'])
+        if not key:
+            print(f"Skip {m['name']} (no key)")
+            continue
+        try:
+            print(f"Testing {m['name']} ({m['model']})...")
+            llm = ChatOpenAI(model=m['model'], openai_api_key=key, openai_api_base=m['base_url'], temperature=0)
+            res = llm.invoke([msg])
+            print(f"Success: {res.content}")
+        except Exception as e:
+            print(f"Fail: {e}")
+
+if __name__ == "__main__":
+    test_vision()