updated tools

app.py

@@ -1,22 +1,15 @@
 import os
 import gradio as gr
-import requests
-import tempfile
-import mimetypes
-import base64
-import json
+import requests, tempfile, base64, json, datetime, re, subprocess, mimetypes, fitz
 import pandas as pd
-import datetime
 from langchain.tools import tool
 from langchain_huggingface import HuggingFaceEndpoint, ChatHuggingFace
 from langchain.agents import initialize_agent, AgentType
 from bs4 import BeautifulSoup
-import base64
 from langchain_openai import ChatOpenAI
-import fitz
+from langchain_community.utilities import ArxivAPIWrapper
+from youtube_transcript_api import YouTubeTranscriptApi
 import yt_dlp
-import re
-import subprocess
 from PIL import Image
 from transformers import pipeline
 
@@ -87,20 +80,6 @@ def current_events_news_search_tool(query: str) -> str:
 # tool.name is set to the function name (e.g., `search_tool`), and 
 # tool.description is set to the docstring of the function  (the triple-quoted string right under def ...) (e.g., "Answer general knowledge or current events queries using DuckDuckGo.").
 
-# --- TOOL 2: Weather Tool (OpenWeatherMap) ---
-@tool
-def get_weather(city: str) -> str:
-    """Get current temperature in Celsius for a city."""
-    import os
-    api_key = os.environ.get("WEATHER_API_KEY")
-    url = f"https://api.openweathermap.org/data/2.5/weather?q={city}&appid={WEATHER_API_KEY}&units=metric"
-    try:
-        resp = requests.get(url, timeout=120)
-        resp.raise_for_status()
-        data = resp.json()
-        return str(round(data["main"]["temp"]))
-    except Exception:
-        return "error"
 
 # --- TOOL 3: Calculator Tool ---
 @tool
@@ -115,45 +94,6 @@ def calculator(expression: str) -> str:
     except Exception:
         return "error"
     
-# --- TOOL 4: Unit Conversion Tool ---
-@tool
-def convert_units(args: str) -> str:
-    """
-    Convert between metric and imperial units (length, mass, temperature).
-    Input format: '<value> <from_unit> to <to_unit>', e.g. '10 meters to feet'
-    """
-    try:
-        parts = args.lower().split()
-        value = float(parts[0])
-        from_unit = parts[1]
-        to_unit = parts[3]
-        conversions = {
-            ("meters", "feet"): lambda v: v * 3.28084,
-            ("feet", "meters"): lambda v: v / 3.28084,
-            ("kg", "lb"): lambda v: v * 2.20462,
-            ("lb", "kg"): lambda v: v / 2.20462,
-            ("celsius", "fahrenheit"): lambda v: v * 9/5 + 32,
-            ("fahrenheit", "celsius"): lambda v: (v - 32) * 5/9,
-        }
-        func = conversions.get((from_unit, to_unit))
-        if func:
-            return str(round(func(value), 2))
-        return "error"
-    except Exception:
-        return "error"
-
-# --- TOOL 5: Date & Time Tool ---
-@tool
-def get_time(input: str) -> str:
-    """Get current UTC time as HH:MM."""
-    return datetime.datetime.utc().strftime("%H:%M")
-
-@tool
-def get_date(input: str) -> str:
-    """Get current date as YYYY-MM-DD."""
-    return datetime.datetime.utc().strftime("%Y-%m-%d")
-
-
 # --- TOOL 6: Wikipedia Summary Tool ---
 @tool
 def wikipedia_and_generalknowledge_search(query: str) -> str:
@@ -195,38 +135,6 @@ def wikipedia_and_generalknowledge_search(query: str) -> str:
     except Exception as e:
         return f"error: {e}"
 
-# --- TOOL 7: Dictionary Tool ---
-@tool
-def dictionary_lookup(word: str) -> str:
-    """Get the definition of an English word using dictionary."""
-    url = f"https://api.dictionaryapi.dev/api/v2/entries/en/{word}"
-    try:
-        resp = requests.get(url, timeout=120)
-        resp.raise_for_status()
-        data = resp.json()
-        return data[0]["meanings"][0]["definitions"][0]["definition"]
-    except Exception:
-        return "error"
-    
-# --- TOOL 8: Currency Conversion Tool ---
-@tool
-def currency_convert(args: str) -> str:
-    """
-    Convert an amount from one currency to another.
-    Input format: '<amount> <from_currency> to <to_currency>', e.g. '100 USD to EUR'
-    """
-    try:
-        parts = args.upper().split()
-        amount = float(parts[0])
-        from_currency = parts[1]
-        to_currency = parts[3]
-        url = f"https://api.exchangerate.host/convert?from={from_currency}&to={to_currency}&amount={amount}"
-        resp = requests.get(url, timeout=120)
-        resp.raise_for_status()
-        data = resp.json()
-        return str(round(data["result"], 2))
-    except Exception:
-        return "error"
 
 # --- TOOL 9: Image Captioning Tool ---
 @tool
@@ -246,33 +154,50 @@ def image_caption(image_url: str) -> str:
 # --- TOOL 10: Optical Character Recognition (OCR) Tool ---
 @tool
 def ocr_image(image_url: str) -> str:
-    """Extract text from an image given its URL."""
-    api_url = "https://api-inference.huggingface.co/models/impira/layoutlm-document-qa"
-    headers = {"Authorization": f"Bearer {HF_ACCESS_KEY}"}
-    payload = {"inputs": {"image": image_url, "question": "What text is in the image?"}}
+    """
+    Extracts all readable text from an image using HuggingFace TrOCR (microsoft/trocr-base-stage1).
+    Input: URL to an image (e.g., PNG or JPG).
+    Output: Recognized text string.
+    """
+    api_url = "https://api-inference.huggingface.co/models/microsoft/trocr-base-stage1"
+    headers = {
+        "Authorization": f"Bearer {HF_ACCESS_KEY}",
+        "Content-Type": "application/json"
+    }
+    payload = {"inputs": image_url}
+
     try:
-        resp = requests.post(api_url, headers=headers, json=payload, timeout=120)
+        resp = requests.post(api_url, headers=headers, json=payload, timeout=60)
         resp.raise_for_status()
         data = resp.json()
-        return data.get("answer", "no_text_found")
-    except Exception:
-        return "error"
+        return data[0]["generated_text"]
+    except Exception as e:
+        return f"OCR error: {e}"
     
 # --- TOOL 11: Image Classification Tool ---
 @tool
-def classify_image(image_url: str) -> str:
-    """Classify the main object or scene in an image given its URL."""
-    api_url = "https://api-inference.huggingface.co/models/google/vit-base-patch16-224"
+def clasify_describe_image(image_url: str) -> str:
+    """
+    Generates a caption describing the contents of an image using HuggingFace (ViT-GPT2). 
+    Use this tool to identify the main subject of an image so that an LLM can use it to answer further.
+    Input: image URL
+    Output: caption like 'A golden retriever lying on a couch.'
+    """
+    api_url = "https://api-inference.huggingface.co/models/nlpconnect/vit-gpt2-image-captioning"
     headers = {"Authorization": f"Bearer {HF_ACCESS_KEY}"}
-    payload = {"inputs": image_url}
+
     try:
-        resp = requests.post(api_url, headers=headers, json=payload, timeout=120)
-        resp.raise_for_status()
-        data = resp.json()
-        return data[0]["label"] if isinstance(data, list) else data.get("label", "no_label")
-    except Exception:
-        return "error"
+        img_resp = requests.get(image_url, timeout=120)
+        img_resp.raise_for_status()
+        image_bytes = img_resp.content
 
+        response = requests.post(api_url, headers=headers, data=image_bytes, timeout=60)
+        response.raise_for_status()
+        result = response.json()
+        return result[0]["generated_text"] if isinstance(result, list) else "no_caption"
+    except Exception as e:
+        return f"caption error: {e}"
+    
 # --- TOOL 12: Web Scraping Tool ---
 @tool
 def URL_scrape_tool(url: str) -> str:
@@ -342,34 +267,69 @@ def python_executor(code: str) -> str:
 @tool
 def python_excel_audio_video_attached_file_tool(input_str: str) -> str:
     """
-    Processes an input attachment (audio, image, video, Excel, or Python .py file) and returns extracted info (text, encoded information, metadata, etc.) to be used by LLM.
-    This function accepts a JSON string 'input_str' with keys: 'file_bytes' (base64), and 'filename'. So input the file and filename as json strings.
+    Accepts a JSON string with one of:
+        • 'file_bytes'  : base-64–encoded bytes  (existing behaviour)
+        • 'file_path'   : local absolute/relative path to a file
+        • 'file_url'    : downloadable URL (e.g. Hugging Face dataset link)
+
+    Keys (at least one bytes / path / url required):
+        • filename  (str) – original name with extension
+        • file_bytes (str, base-64)    – optional
+        • file_path  (str)             – optional
+        • file_url   (str)             – optional
+
+    Returns: textual summary / preview ready for the LLM.
     """
-    import pandas as pd
 
+    # ---------- 1. Parse JSON ------------------------------------------------
     try:
-        # Extract only the JSON object from the input string
+        # Robustly pull out the first {...} block even if extra tokens are around it
         match = re.search(r'(\{.*\})', input_str, re.DOTALL)
-        if match:
-            input_str = match.group(1)
-        data = json.loads(input_str)
-        file_bytes = base64.b64decode(data["file_bytes"])
-        filename = data["filename"]
+        payload = json.loads(match.group(1) if match else input_str)
     except Exception as e:
-        return f"error: {e}"
+        return f"error: Could not parse JSON → {e}"
+
+    filename  = payload.get("filename")
+    b64_data  = payload.get("file_bytes")
+    file_path = payload.get("file_path")
+    file_url  = payload.get("file_url")
+
+    if not filename:
+        return "error: 'filename' is required."
+
+    # ---------- 2. Acquire raw bytes ----------------------------------------
+    try:
+        if b64_data:                                        # inline bytes
+            file_bytes = base64.b64decode(b64_data)
+
+        elif file_path and os.path.exists(file_path):       # local path
+            with open(file_path, "rb") as f:
+                file_bytes = f.read()
+
+        elif file_url:                                      # remote URL
+            # stream to avoid loading huge files into memory at once
+            r = requests.get(file_url, timeout=60, stream=True)
+            r.raise_for_status()
+            file_bytes = r.content
+
+        else:
+            return "error: Provide 'file_bytes', 'file_path', or 'file_url'."
+    except Exception as e:
+        return f"error: Could not load file → {e}"
 
     # Detect file type
     mime_type, _ = mimetypes.guess_type(filename)
+    # fallback for common extensions if guess_type fails
     if not mime_type:
-        # Fallback for .py and .csv files
-        if filename.lower().endswith(".py"):
-            mime_type = "text/x-python"
-        elif filename.lower().endswith(".csv"):
-            mime_type = "text/csv"
-        elif filename.lower().endswith((".xls", ".xlsx")):
-            mime_type = "application/vnd.ms-excel"
-        else:
-            return "error: Could not determine file type. Skip the file"
+        ext = filename.lower()
+        mime_type = (
+            "text/x-python"          if ext.endswith(".py")  else
+            "text/csv"               if ext.endswith(".csv") else
+            "application/vnd.ms-excel" if ext.endswith((".xls", ".xlsx")) else
+            None
+        )
+    if not mime_type:
+        return "error: Could not determine file type. Skip the file."
 
     # Handle audio files
     if mime_type.startswith("audio"):
@@ -390,7 +350,23 @@ def python_excel_audio_video_attached_file_tool(input_str: str) -> str:
 
     # Handle image files
     elif mime_type.startswith("image"):
-        image_b64 = base64.b64encode(file_bytes).decode()
+        # image_b64 = base64.b64encode(file_bytes).decode()
+        api_url = "https://api-inference.huggingface.co/models/nlpconnect/vit-gpt2-image-captioning"
+        headers = {"Authorization": f"Bearer {os.getenv('HF_ACCESS_KEY', '')}"}
+        try:
+            resp = requests.post(api_url, headers=headers, data=file_bytes, timeout=60)
+            resp.raise_for_status()
+            result = resp.json()
+            if isinstance(result, list) and result and "generated_text" in result[0]:
+                caption = result[0]["generated_text"]
+            else:
+                caption = "no_caption"
+
+            # Optionally also include base-64 so the LLM can refer to the raw image
+            b64 = base64.b64encode(file_bytes).decode()
+            return f"Image caption: {caption}\nAttached image (base64): {b64}"
+        except Exception as e:
+            return f"caption error: {e}"
         return f"Attached image (base64): {image_b64}"
 
     # Handle video files (extract audio, then transcribe)
@@ -402,7 +378,7 @@ def python_excel_audio_video_attached_file_tool(input_str: str) -> str:
                 video_path = tmp_video.name
 
             audio_path = video_path + ".wav"
-            import subprocess
+            # import subprocess
             subprocess.run([
                 "ffmpeg", "-i", video_path, "-vn", "-acodec", "pcm_s16le", "-ar", "16000", "-ac", "1", audio_path
             ], check=True)
@@ -464,70 +440,18 @@ def python_excel_audio_video_attached_file_tool(input_str: str) -> str:
 
 # --- TOOL 16: Research Paper Info Extraction Tool ---
 @tool
-def search_and_extract_research_paper_info(query: str) -> str:
+def research_paper_search(query: str) -> str:
     """
-    Searches for research and online papers/journals using the Semantic Scholar API.
-    Input: A search query (e.g., topic, paper title, or keywords).
-    Output: A summary with title, authors, abstract, and a longer excerpt from the main sections of the top result.
+    Search arXiv for journals/research/technical papers matching a query. 
+    Returns top results including title, authors, abstract, and PDF link.
     """
-    try:
-        # Search for papers using Semantic Scholar API
-        search_url = "https://api.semanticscholar.org/graph/v1/paper/search"
-        params = {
-            "query": query,
-            "limit": 1,
-            "fields": "title,authors,abstract,url,openAccessPdf"
-        }
-        resp = requests.get(search_url, params=params, timeout=120)
-        resp.raise_for_status()
-        data = resp.json()
-        if not data.get("data"):
-            return "No papers found for this query."
-        paper = data["data"][0]
-        title = paper.get("title", "")
-        authors = ", ".join([a["name"] for a in paper.get("authors", [])])
-        abstract = paper.get("abstract", "")
-        paper_url = paper.get("url", "")
-        pdf_url = paper.get("openAccessPdf", {}).get("url")
-        if not pdf_url:
-            return f"Paper found: {title}\nAuthors: {authors}\nAbstract: {abstract}\nURL: {paper_url}\n(No open access PDF available.)"
-
-        # Download the PDF
-        pdf_resp = requests.get(pdf_url, timeout=120)
-        pdf_resp.raise_for_status()
-        with tempfile.NamedTemporaryFile(delete=False, suffix=".pdf") as tmp_pdf:
-            tmp_pdf.write(pdf_resp.content)
-            tmp_pdf.flush()
-            pdf_path = tmp_pdf.name
-
-        # Extract text from PDF
-        doc = fitz.open(pdf_path)
-        full_text = ""
-        for page in doc:
-            full_text += page.get_text("text") + "\n"
-
-        # Extract a longer excerpt from the main sections (e.g., Introduction + first 2000 chars)
-        lines = full_text.splitlines()
-        main_sections = ""
-        in_main = False
-        for line in lines:
-            if "introduction" in line.lower():
-                in_main = True
-            if in_main:
-                main_sections += line.strip() + " "
-                if len(main_sections) > 2000:
-                    break
-
-        summary = (
-            f"Title: {title}\n"
-            f"Authors: {authors}\n"
-            f"Abstract: {abstract}\n"
-            f"URL: {paper_url}\n"
-            f"Main Sections (excerpt): {main_sections.strip() if main_sections else full_text[:2000]}"
-        )
-        return summary if summary.strip() else "No information extracted."
-    except Exception as e:
-        return f"error: {e}"
+    wrapper = ArxivAPIWrapper(
+        top_k_results=2,                  # how many papers to return
+        doc_content_chars_max=2000        # max chars of abstract to show
+    )
+    
+    results_text = wrapper.run(query)
+    return results_text
     
 
 # --- TOOL 17:Tool for sports, awards, competitions etc. ---
@@ -599,6 +523,27 @@ def sports_awards_historicalfacts_tool(query: str) -> str:
     except Exception as e:
         return f"error: {e}"
 
+# --- TOOL 17: YouTube Transcript Tool ---
+@tool
+def youtube_transcript_tool(video_url: str) -> str:
+    """
+    Get transcript (if available) for a YouTube video without downloading audio.
+    Works only if subtitles or auto-captions exist.
+    """
+    try:
+        # Extract video ID
+        match = re.search(r"(?:v=|youtu\.be/)([a-zA-Z0-9_-]{11})", video_url)
+        if not match:
+            return "Invalid YouTube URL."
+        video_id = match.group(1)
+
+        transcript = YouTubeTranscriptApi.get_transcript(video_id)
+        full_text = " ".join([chunk['text'] for chunk in transcript])
+        return full_text[:5000]  # truncate to keep LLM input manageable
+    except Exception as e:
+        return f"Transcript error: {e}"
+    
+
 # --- TOOL 18: YouTube Transcript Tool ---
 @tool
 def video_url_to_transcript_tool(media_url: str) -> str:
@@ -749,24 +694,27 @@ tools_list = [
     python_excel_audio_video_attached_file_tool,
     wikipedia_and_generalknowledge_search,
     # sports_awards_historicalfacts_tool,
-    search_and_extract_research_paper_info,
+    research_paper_search,
     python_executor,
     # get_weather,
     # calculator,
     # convert_units,
     # get_time,
     # get_date,
-    dictionary_lookup,
+    # dictionary_lookup,
     # currency_convert,
     # image_caption,
     # ocr_image,
     # classify_image,
     current_events_news_search_tool,
+    ocr_image,
+    clasify_describe_image,
     URL_scrape_tool,
     # audio_url_to_text, 
     # sports_awards_historicalfacts_tool,
-    video_url_to_transcript_tool,
-    # max_object_in_video,
+    youtube_transcript_tool,
+    # video_url_to_transcript_tool,
+    max_object_in_video,
 ]
 
 tool_descriptions = "\n".join(f"- {tool.name}: {tool.description}" for tool in tools_list)
@@ -777,7 +725,7 @@ tool_descriptions = "\n".join(f"- {tool.name}: {tool.description}" for tool in t
 # --- System Prompt for the Agent ---
 
 system_prompt = f"""
-You are a general AI assistant, who can answer about general knowledge, historical facts, and also can analyze audios, images and videos. You should think through the input question step-by-step and use tools if needed.
+You are a general AI assistant, who can answer about general knowledge, historical facts, and also can analyze audios, images, and videos. You should think through the input question step-by-step and use tools if needed.
 
 Use this reasoning format repeatedly:
 Thought: (what you think is happening or what you want to do next)
@@ -785,26 +733,17 @@ Action: (the tool to use, if needed)
 Action Input: (input to the tool)
 Observation: (result of the tool call)
 
-Repeat this process as needed. ONLY AFTER finishing your reasoning and tool use, provide: YOUR FINAL ANSWER 
-YOUR FINAL ANSWER should be just a number, string, or comma-separated list with no explanations, no thoughts, just the answer
+Repeat this process as needed. ONLY AFTER finishing your reasoning and/or tool use, provide: YOUR FINAL ANSWER 
+YOUR FINAL ANSWER should be just a number, string, or comma-separated list with no explanations and no thoughts, but just the answer
 
 You also have access to a set of tools, which you can use to answer the question. The available tools are:
 {tool_descriptions}
 
 If the question is related to sports, awards, historical facts or similar topic that can be answered from wikipedia, you should use the 'wikipedia_and_generalknowledge_search'. 
-If the question is based on current events or news kind, then you can utilize the tool 'current_events_news_search_tool' to fetch relevant page information and answer from it. 
+If the question is about current events or news or similar current affairs category, you can utilize the tool 'current_events_news_search_tool' to fetch relevant page information and answer from it. 
 If the tool returns a long text, table, or list, extract only the most relevant information/paragraphs or data from which you can derive the answer, and return that as your final answer.
 You must not use multiple tools in a single call. Don't hallucinate.
 
-Instructions to follow for YOUR FINAL ANSWER:
-Guidance:
-- Prefer reasoning and tool use before jumping to final answer.
-- Only use one tool per iteration.
-- If the question relates to historical data or general knowledge, use 'wikipedia_and_generalknowledge_search'.
-- If it relates to recent news, use 'current_events_news_search_tool'.
-- If tool output is long, extract only what is relevant and use it to answer the question.
-- Never hallucinate. If you cannot answer even after 12 iterations, return YOUR FINAL ANSWER as: no_answer
-- Don't include explanations, thoughts, or tool calls in YOUR FINAL ANSWER.
 
 **Examples:**
 Q: Which country had the least number of athletes at the 1928 Summer Olympics?
@@ -818,8 +757,9 @@ Q: What are the top 3 programming languages?
 Thought: This is common knowledge.
 YOUR FINAL ANSWER: Python, JavaScript, Java
 
-If after 12 iterations also a tool usage is not useful then try to answer directly based on your knowledge without any hallucination. If you cannot answer then just say "no_answer" as YOUR FINAL ANSWER. 
+If even after 12 iterations, a tool usage is not useful then try to answer directly based on your knowledge without any hallucination. If you cannot answer then just say "no_answer" as YOUR FINAL ANSWER. 
 """
+
 # If your final answer is something like 'there were 5 studio albums published between 2000 and 2009' then modify YOUR FINAL ANSWER as: '5' 
 # If your final answer is something like 'b, e' then YOUR FINAL ANSWER be: 'b, e'
 # For each question, follow this format:

requirements.txt

@@ -13,4 +13,6 @@ mimetype
 PyMuPDF
 yt_dlp
 pandas
-pillow
+pillow
+arxiv
+youtube-transcript-api