add youtube transcription

agents/search_agent.py

@@ -16,13 +16,16 @@ class SearchAgent:
 
         state = workflow.invoke({
             "messages":messages,
+            "question": question,
         }, config={"callbacks": [langfuse_handler]})
         
-        print(state["external_information"])
         return state["answer"]
     
 if __name__ == "__main__":
-    question = "How many studio albums were published by Mercedes Sosa between 2000 and 2009 (included)? You can use the latest 2022 version of english wikipedia."
+    #question = "In the video https://www.youtube.com/watch?v=L1vXCYZAYYM, what is the highest number of bird species to be on camera simultaneously?"
+    question = """Examine the video at https://www.youtube.com/watch?v=1htKBjuUWec.
+
+What does Teal'c say in response to the question "Isn't that hot?"""
     agent = SearchAgent()
     submit_answer = agent(question)
     

graphs/evaluation.py

@@ -1,14 +1,17 @@
 from models.models import  groq_model, anthropic_model
-from tools.search import  taivily_search, serper_search
+from tools import  taivily_search, serper_search, execute_code, get_youtube_transcript
 from langgraph.graph import StateGraph, START, END, MessagesState
+from langgraph.types import Command
 from langchain_core.messages import HumanMessage, SystemMessage
 from typing import List, TypedDict
 from langgraph.prebuilt import ToolNode
-
+from IPython.display import Image
 
 tools = [
-    #taivily_search,
+    taivily_search,
     serper_search,
+    get_youtube_transcript,
+    execute_code
 ]
 
 class EvaluationState(TypedDict):
@@ -19,6 +22,8 @@ class EvaluationState(TypedDict):
     answer: str
     external_information: str
     has_enough_information: bool
+    is_valid_answer: bool
+    step_counter: dict[str, int]
     
 bound_model_llama = groq_model.bind_tools(tools)
 bound_model_antrhropic = anthropic_model.bind_tools(tools)
@@ -27,8 +32,7 @@ def call_node(state: EvaluationState):
     """
     This node call the model with the question and the tools
     """
-    question = state["messages"][-1].content
-    state["question"] = question
+    print(state["question"])
     response = bound_model_llama.invoke(state["messages"])
     
     state["messages"].append(response)
@@ -37,24 +41,23 @@ def call_node(state: EvaluationState):
 tool_node = ToolNode(tools)
 
 
-def parse_response(state: EvaluationState):
+def answer_question(state: EvaluationState):
     """
-    Parse the response from the model and return the final answer
+    This node get the context information and call the model to get the answer.
     """
-    prompt = f"""I will ask you a question. Report your thoughts, and finish with only YOUR FINAL ANSWER.
-                                 YOUR FINAL ANSWER should be a number OR as few words as possible OR a comma separated list of numbers and/or strings. 
-                                 If you are asked for a number, don't use comma to write your number neither use units such as $ or percent sign unless specified otherwise.
-                                 If you are asked for a string, don't use articles, neither abbreviations (e.g. for cities), and write the digits in plain text unless specified otherwise.
-                                 If you are asked for a comma separated list, apply the above rules depending of whether the element to be put in the list is a number or a string.
-                                ---question---
-                                {state["question"]}
-                                ---relevant information---
-                                {state["external_information"]}
-                                ---answer---
-                                """
-    
-    
-    response  = groq_model.invoke(prompt)
+    prompt = f"""## Instruction \n I will ask you a question. Report your thoughts, and finish with only YOUR FINAL ANSWER. 
+YOUR FINAL ANSWER should be a number OR as few words as possible OR a comma separated list of numbers and/or strings. 
+If you are asked for a number, don't use comma to write your number neither use units such as $ or percent sign unless specified otherwise.
+If you are asked for a string, don't use articles, neither abbreviations (e.g. for cities), and write the digits in plain text unless specified otherwise.
+If you are asked for a comma separated list, apply the above rules depending of whether the element to be put in the list is a number or a string.
+## Question
+{state["question"]}
+## Relevant information
+{state["external_information"]}
+## answer"""
+    
+    
+    response  = anthropic_model.invoke(prompt)
     state["messages"].append(response)
     state["answer"] = response.content
     return state
@@ -69,30 +72,114 @@ def map_answer(state: EvaluationState):
         "answer": answer.content
     }
 
+def validator(state: EvaluationState):
+    """
+    Validate if the answer fills the requirements
+    """
+    # Initialize or update validator step counter
+    if "step_counter" not in state:
+        state["step_counter"] = { "validator": 0}
+    
+    # Increment the validator step counter
+    state["step_counter"]["validator"] = state["step_counter"].get("validator", 0) + 1
+    
+    # Check if we've hit the validator recursion limit
+    if state["step_counter"]["validator"] >= 3:  # Smaller limit for validator recursion
+        print("Validator recursion limit reached. Accepting current answer format.")
+        state["is_valid_answer"] = True
+        return state
+    
+    answer = state["answer"]
+    result = anthropic_model.invoke(f"Validate if the answer fits the next requirements: \n\n{answer}\n\nThe answer should be a number, string or a list of numbers and/or strings. If the answer fits the requirements, return 'yes', otherwise return 'no'.")
+    is_valid_answer = result.content.startswith("yes")  
+    state["is_valid_answer"] = is_valid_answer
+    
+    return state
+
+def evaluator(state: EvaluationState):
+    """
+    Evaluate if it is needed more infomation to resolve the question.
+    """
+    if "step_counter" not in state:
+        state["step_counter"] = { "evaluator": 0}
+    
+    state["step_counter"]["evaluator"] = state["step_counter"].get("evaluator", 0) + 1
+    
+    total_iterations =  state["step_counter"].get("evaluator", 0)
+    if total_iterations >= 5:  # Using higher threshold for combined count
+        state["has_enough_information"] = True
+        return state
+    
+    prompt = f"""Does the context information are enough to resolve the answer? \n # Context information \n {state["external_information"]} \n # Question \n {state["question"]} \n If the context information is enough to resolve the question, return 'yes', otherwise return what is missing."""
+    result = anthropic_model.invoke(prompt)
+    has_enough_information = result.content.startswith("yes")
+    state["has_enough_information"] = has_enough_information
+    
+    if not has_enough_information:
+        # Only update messages and external information if we need more info
+        state["messages"] = [SystemMessage(content=result.content)]
+        state["external_information"] = f"{state['external_information']}\n\n---\n\n{result.content}"
+    
+    return state
+        
 def build_workflow():
     """
-    Build search  workflow
+    Build search workflow with conditional edge for evaluation
     """
     workflow = StateGraph(EvaluationState)
     workflow.add_node("agent", call_node)
     workflow.add_node("action", tool_node)
-    workflow.add_node("parse_response", parse_response)
+    workflow.add_node("answer_question", answer_question)
     workflow.add_node("map_answer", map_answer)
-
-    workflow.add_edge(START,"agent")
+    workflow.add_node("evaluator", evaluator)
+    workflow.add_node("validator", validator)
+    
+    # Define edges
+    workflow.add_edge(START, "agent")
     workflow.add_edge("agent", "action")
-    workflow.add_edge("action", "parse_response")
-    workflow.add_edge("parse_response", "map_answer")
-    workflow.add_edge("map_answer", END)
+    workflow.add_edge("action", "evaluator")
+    
+    # Explicit conditional edges from evaluator
+    def route_evaluator(state):
+        if state["has_enough_information"]:
+            return "answer_question"
+        else:
+            return "agent"
+    
+    workflow.add_conditional_edges("evaluator", route_evaluator,{"answer_question":"answer_question","agent":"agent"})
+    
+    # Connect answer_question to map_answer
+    workflow.add_edge("answer_question", "map_answer")
+    workflow.add_edge("map_answer", "validator")
+    
+    # Explicit conditional edges from validator
+    def route_validator(state):
+        if state["is_valid_answer"]:
+            return END
+        else:
+            return "map_answer"
+    
+    workflow.add_conditional_edges("validator", route_validator, {"map_answer":"map_answer", END:END})
+    
+    # Check if we need to manually add the edges for visualization
+    try:
+        # These are just for visualization and may not affect actual execution
+        workflow._graph.add_edge("evaluator", "agent", condition="needs more info")
+        workflow._graph.add_edge("evaluator", "answer_question", condition="has enough info")
+        workflow._graph.add_edge("validator", "map_answer", condition="invalid answer")
+        workflow._graph.add_edge("validator", END, condition="valid answer")
+    except:
+        # Skip if this approach doesn't work with current LangGraph version
+        pass
     
     return workflow.compile()
 
+
 """ if __name__ == "__main__":
-    question = "How many studio albums were published by Mercedes Sosa between 2000 and 2009 (included)? You can use the latest 2022 version of english wikipedia."
     # Build the graph
     graph = build_workflow()
-    # Run the graph
-    messages = [HumanMessage(content=question)]
-    messages = graph.invoke({"messages": messages})
-    for m in messages["messages"]:
-        m.pretty_print() """
+    
+    # Get the Mermaid diagram as text
+    mermaid_text = graph.get_graph().draw_mermaid()
+    
+    print(mermaid_text) """

graphs/output/workflow_graph.mmd

@@ -0,0 +1,21 @@
+---
+config:
+  flowchart:
+    curve: linear
+---
+graph TD;
+	__start__(<p>__start__</p>)
+	agent(agent)
+	action(action)
+	answer_question(answer_question)
+	map_answer(map_answer)
+	evaluator(evaluator)
+	validator(validator)
+	__end__(<p>__end__</p>)
+	__start__ --> agent;
+	action --> evaluator;
+	agent --> action;
+	evaluator --> __end__;
+	classDef default fill:#f2f0ff,line-height:1.2
+	classDef first fill-opacity:0
+	classDef last fill:#bfb6fc

models/models.py

@@ -5,8 +5,8 @@ from dotenv import load_dotenv
 load_dotenv()
  
 anthropic_model = ChatAnthropic(
-    model="claude-3-5-haiku-latest",
-    temperature=0
+    model="claude-3-7-sonnet-20250219",
+    temperature=0.4
 )
 
 groq_model = ChatGroq(

requirements.txt

@@ -9,11 +9,17 @@ arxiv==2.2.0
 attrs==25.3.0
 backoff==2.2.1
 beautifulsoup4==4.13.4
+bytecode==0.16.2
 certifi==2025.4.26
 charset-normalizer==3.4.2
 click==8.1.8
 dataclasses-json==0.6.7
+ddtrace==3.6.0
+Deprecated==1.2.18
 distro==1.9.0
+e2b==1.4.0
+e2b-code-interpreter==1.5.0
+envier==0.6.1
 fastapi==0.115.12
 feedparser==6.0.11
 ffmpy==0.5.0
@@ -31,6 +37,7 @@ httpx==0.28.1
 httpx-sse==0.4.0
 huggingface-hub==0.30.2
 idna==3.10
+importlib_metadata==8.6.1
 Jinja2==3.1.6
 jiter==0.9.0
 jsonpatch==1.33
@@ -54,12 +61,14 @@ mdurl==0.1.2
 multidict==6.4.3
 mypy_extensions==1.1.0
 numpy==2.2.5
+opentelemetry-api==1.32.1
 orjson==3.10.18
 ormsgpack==1.9.1
 packaging==24.2
 pandas==2.2.3
 pillow==11.2.1
 propcache==0.3.1
+protobuf==5.29.4
 pydantic==2.11.4
 pydantic-settings==2.9.1
 pydantic_core==2.33.2
@@ -97,6 +106,9 @@ uvicorn==0.34.2
 websockets==15.0.1
 wikipedia==1.4.0
 wrapt==1.17.2
+xmltodict==0.14.2
 xxhash==3.5.0
 yarl==1.20.0
+yt-dlp==2024.4.9
+zipp==3.21.0
 zstandard==0.23.0

tools/__init__.py

@@ -0,0 +1,4 @@
+from tools.search import taivily_search, serper_search
+from tools.sandbox import execute_code, get_youtube_transcript
+
+__all__ = ["taivily_search", "serper_search", "execute_code", "get_youtube_transcript"]

tools/sandbox.py

@@ -0,0 +1,151 @@
+from langchain_core.tools import tool
+from typing import Annotated
+from typing_extensions import Annotated
+from langchain_core.tools.base import InjectedToolCallId
+from langchain_core.runnables import RunnableConfig
+from langgraph.types import Command
+from langchain_core.messages import ToolMessage
+import os
+from dotenv import load_dotenv
+import json
+import asyncio
+import tempfile
+from pathlib import Path
+import yt_dlp
+from e2b_code_interpreter import Sandbox
+
+load_dotenv()
+
+@tool
+def execute_code(code: str, tool_call_id: Annotated[str, InjectedToolCallId], config: RunnableConfig) -> Command:
+    """
+    Execute code in a secure E2B sandbox environment.
+    
+    Args:
+        code: The code to execute. Should be Python code without the triple backticks.
+    """
+    try:
+        loop = asyncio.get_event_loop()
+    except RuntimeError:
+        loop = asyncio.new_event_loop()
+        asyncio.set_event_loop(loop)
+    
+    result = loop.run_until_complete(_execute_code_in_sandbox(code, os.getenv("E2B_API_KEY")))
+    
+    formatted_result = f"""# Code Execution Results
+## Code
+```python
+{code}
+```
+## Output
+```
+{result['stdout']}
+```
+## Errors
+```
+{result['stderr']}
+```
+"""
+    
+    external_information = f"{config.get('external_information', '')}\n---\n# Code Execution Results \n{formatted_result}"
+    return Command(
+        update={
+            "external_information": external_information,
+            "messages": [ToolMessage(content=formatted_result, tool_call_id=tool_call_id)]
+        }
+    )
+    
+async def _execute_code_in_sandbox(code: str, api_key: str):
+    """Execute code in E2B sandbox and return the results."""
+    sbx = Sandbox()
+    execution = sbx.run_code(code)
+    
+    files = sbx.files.list("/")
+    
+    return {
+        "stdout": execution.stdout,
+        "stderr": execution.stderr,
+        "files": files
+    }
+
+@tool
+def get_youtube_transcript(url: str, tool_call_id: Annotated[str, InjectedToolCallId] = None, config: RunnableConfig = None) -> Command | str:
+    """
+    This tool extracts the transcript text from YouTube videos, returns the transcript as a string.
+    Args:
+        url: The YouTube video URL.
+    Returns:
+        The transcript as a string, or an error message if the transcript couldn't be obtained
+    """
+    temp_dir = tempfile.mkdtemp()
+    current_dir = os.getcwd()
+    
+    try:
+        os.chdir(temp_dir)
+         
+        ydl_opts = {
+            'writesubtitles': True,       
+            'writeautomaticsub': True,     
+            'subtitleslangs': ['en'],    
+            'skip_download': True,        
+            'outtmpl': 'subtitle',        
+        }
+        
+        with yt_dlp.YoutubeDL(ydl_opts) as ydl:
+            ydl.extract_info(url, download=True)
+        
+        subtitle_content = ""
+        subtitle_files = list(Path(temp_dir).glob("*.vtt")) + list(Path(temp_dir).glob("*.srt"))
+        
+        if subtitle_files:
+            with open(subtitle_files[0], 'r', encoding='utf-8') as f:
+                subtitle_content = f.read()
+            
+            lines = subtitle_content.split('\n')
+            cleaned_lines = []
+            for line in lines:
+                if line.strip() and not line.strip().isdigit() and not '-->' in line and not line.startswith('WEBVTT'):
+                    cleaned_lines.append(line)
+            subtitle_content = '\n '.join(cleaned_lines)
+        else:
+            subtitle_content = "Error: No subtitles found for this video."
+            
+    except Exception as e:
+        subtitle_content = f"Error retrieving YouTube transcript: {str(e)}"
+    finally:
+        os.chdir(current_dir)
+        
+        try:
+            for file in os.listdir(temp_dir):
+                os.remove(os.path.join(temp_dir, file))
+            os.rmdir(temp_dir)
+        except:
+            pass
+    
+    external_information= f"{config.get('external_information', '')}\n---\n# Youtube transcript \n{subtitle_content}"
+    return Command(
+            update={
+                "external_information": external_information,
+                "messages": [ToolMessage(content=subtitle_content, tool_call_id=tool_call_id)]
+            }
+        )
+
+""" if __name__ == "__main__":
+    # Simple test: print "Hello World"
+    test_code = "print(\"Hello World\")"
+
+    # Build a minimal RunnableConfig with no external information
+    config = RunnableConfig(**{"external_information": ""})
+
+    # Execute the test code
+    # Call the underlying function to bypass the BaseTool wrapper
+    cmd: Command = execute_code.func(
+        test_code,
+        "test-call",
+        config,
+    )
+
+    # Print the output from the sandbox execution
+    updates = getattr(cmd, 'update', {}) or {}
+    for msg in updates.get('messages', []):
+        print(msg.content) """