Upload 9 files

agent.py

@@ -0,0 +1,126 @@
+from typing import Optional
+
+import langchain
+from dotenv import load_dotenv
+from langchain_core.prompts import PromptTemplate
+from langchain import chains
+from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler
+from pydantic import ValidationError
+from langchain.agents import AgentExecutor
+from langchain.agents.mrkl.base import ZeroShotAgent
+from prompts import FORMAT_INSTRUCTIONS, QUESTION_PROMPT, QUESTION_PROMPT1, SUFFIX
+from tools import make_tools
+ 
+from rmrkl import ChatZeroShotAgent, RetryAgentExecutor
+ 
+import base64
+from io import BytesIO
+from PIL import Image
+from IPython.display import HTML, display
+from langchain_openai import ChatOpenAI , OpenAI
+
+def convert_to_base64(pil_image):
+    buffered = BytesIO()
+    pil_image.save(buffered, format="PNG")
+    img_str = base64.b64encode(buffered.getvalue()).decode("utf-8")
+    return img_str
+
+def _make_llm(model, temp, api_key, streaming: bool = False):
+    if model.startswith("claude") or model.startswith("gpt-3"):
+        llm = OpenAI(
+       temperature=temp,
+       model_name=model,
+max_tokens = 5000,
+       openai_api_key=api_key,
+       base_url="https://www.dmxapi.com/v1"
+   )
+    elif model.startswith("gpt-4o-2024-11-20") or model.startswith("deepseek"):
+        llm = ChatOpenAI(model=model,
+            temperature = 0.1,
+            
+            timeout=1000,
+            
+            callbacks=[StreamingStdOutCallbackHandler()],
+            openai_api_key=api_key,base_url="https://www.dmxapi.com/v1"
+            )
+   
+    else:
+        raise ValueError(f"Invalid model name: {model}")
+    return llm
+
+
+class TeLLAgent:
+    def __init__(
+        self,
+        tools=None,
+        model1: str = "deepseek-ai/DeepSeek-R1",
+        model2: str = "deepseek-ai/DeepSeek-V3",
+        tools_model="gpt-4o-2024-11-20",
+        temp=0.1,
+        max_iterations=50,
+        verbose=True,
+        streaming: bool = True,
+        openai_api_key= None,
+        api_keys: str = {},
+        file_path: str= r"...",
+        image_path: str = r"..."
+    ):
+        """Initialize agent."""
+        self.file_path = file_path
+        self.image_path = image_path
+        load_dotenv()
+        try:
+            self.llm1 = _make_llm(model1, temp, openai_api_key, streaming)
+            self.llm2 = _make_llm(model2, temp, openai_api_key, streaming)
+        except ValidationError:
+            raise ValueError("Invalid OpenAI API key")
+
+        if tools is None:
+            api_keys["OPENAI_API_KEY"] = 'sk-itPrztYm9F6XZZpsBMJB9O7Vq0pYUABVVBSoThuBxEGTnDik'
+            tools_llm = _make_llm(tools_model, temp, openai_api_key, streaming)
+            tools = make_tools(tools_llm, api_keys=api_keys, verbose=verbose, image_path = image_path, file_path = file_path)
+ 
+        # Initialize agent
+        self.agent_executor1 = RetryAgentExecutor.from_agent_and_tools(
+            tools=tools,
+            agent=ChatZeroShotAgent.from_llm_and_tools(
+                self.llm1,
+                tools,
+                suffix=SUFFIX,
+                format_instructions=FORMAT_INSTRUCTIONS,
+                question_prompt=QUESTION_PROMPT1, return_intermediate_steps=True ,handle_parsing_errors=True 
+            ),
+            verbose=True,
+            max_iterations=1 , return_intermediate_steps=True, handle_parsing_errors=True 
+        )
+        self.agent_executor2 = RetryAgentExecutor.from_agent_and_tools(
+            tools=tools,
+            agent=ChatZeroShotAgent.from_llm_and_tools(
+                self.llm2,
+                tools,
+                suffix=SUFFIX,
+                format_instructions=FORMAT_INSTRUCTIONS,
+                question_prompt=QUESTION_PROMPT,return_intermediate_steps=True ,handle_parsing_errors=True 
+            ),
+            verbose=True,
+            max_iterations=max_iterations , return_intermediate_steps=True  ,handle_parsing_errors=True  
+        )
+ 
+
+    def run(self, prompt):
+         
+        outputs = self.agent_executor1.invoke( {"input": prompt})
+        if outputs["intermediate_steps"] ==[]:
+            prompt = str(' ' +outputs["input"]+ ' ' + outputs["output"].split('Action:')[0] )
+            outputs = self.agent_executor2.invoke( {"input":prompt })            
+        else: 
+            prompt = str(' ' + outputs["input"] + ' ' + outputs["intermediate_steps"][0][0].log.split('Action:')[0])
+            outputs = self.agent_executor2.invoke( {"input": prompt})
+        return outputs 
+
+if __name__ == '__main__':
+        chem_model = TeLLAgent( temp=0.1, streaming=False,
+                           openai_api_key =r'sk-itPrztYm9F6XZZpsBMJB9O7Vq0pYUABVVBSoThuBxEGTnDik', 
+                           image_path= r"C:\Users\BM109X32G-10GPU-02\Pictures\1735356359936.jpg"
+                           )
+        chem_model.run(r"""what is Y20""")

app.py

@@ -0,0 +1,219 @@
+import os
+import asyncio
+# Init with fake key
+if 'OPENAI_API_KEY' not in os.environ:
+    os.environ['OPENAI_API_KEY'] = 'none'
+if os.name == 'nt':
+    asyncio.set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy())
+    
+import openai
+import pandas as pd
+import streamlit as st
+from IPython.core.display import HTML
+from PIL import Image
+from  agent import TeLLAgent, make_tools
+from streamlit_callback_handler import \
+    StreamlitCallbackHandlerChem
+import base64 
+import pandas as pd
+from dotenv import load_dotenv
+from langchain_openai import ChatOpenAI , OpenAI
+import base64
+from io import BytesIO
+from PIL import Image
+import tempfile
+ 
+
+def convert_to_base64(pil_image):
+    buffered = BytesIO()
+    pil_image.save(buffered, format="PNG")
+    img_str = base64.b64encode(buffered.getvalue()).decode("utf-8")
+    return img_str
+
+def oai_key_isvalid(api_key):
+    """Check if a given OpenAI key is valid"""
+    try:
+        llm = ChatOpenAI(openai_api_key = api_key, base_url="https://www.dmxapi.com/v1/")
+        out = llm.invoke("This is a test")
+        return True
+    except:
+        return False
+    
+load_dotenv()
+ss = st.session_state
+ss.prompt = None
+ 
+# Set width of sidebar
+st.markdown(
+    """
+    <style>
+    [data-testid="stSidebar"][aria-expanded="true"]{
+        min-width: 450px;
+        max-width: 450px;
+    }
+    """,
+    unsafe_allow_html=True,
+)
+
+
+def instantiate_agent(model,file_path = '...',
+        image_path ='...'):
+    ss.agent = TeLLAgent(
+        model=model,
+        tools_model=model,
+        temp=0.1,
+        openai_api_key=ss.get('api_key')    ,        file_path = file_path,
+                image_path =image_path
+      
+    )
+    return ss.agent
+
+instantiate_agent('gpt-4o-2024-11-20')
+tools = ss.agent.agent_executor.tools
+
+tool_list = pd.Series(
+    {f"✅ {t.name}":t.description for t in tools}
+).reset_index()
+tool_list.columns = ['Tool', 'Description']
+
+def on_api_key_change():
+    api_key = ss.get('api_key') or os.getenv('OPENAI_API_KEY')
+     
+    # Check if key is valid
+    if not oai_key_isvalid(api_key):
+        st.write("Please input a valid OpenAI API key.")
+
+def run_prompt(prompt, file_path = '...', image_path = '...'):
+    agent = instantiate_agent(ss.get('model_select'),file_path = file_path, image_path =image_path)
+    st.chat_message("user").write(prompt)
+    with st.chat_message("assistant") :
+        try:
+           
+           response = agent.run(prompt)
+           if ss.get('file_type') == 'CSV (.csv)':
+               try: 
+                   fx = pd.DataFrame(list(response))
+                   st.markdown(":red[Prediction finished! ]")	
+                   st.download_button( "⬇️Download  the predicted files as .csv", fx.to_csv(), "predict results.csv", use_container_width=True)
+               except:
+                   st.write(response)
+           else:
+                st.write(response)
+        except openai.AuthenticationError:
+            st.write("Please input a valid OpenAI API key")
+        except openai.APIError:
+            # Handle specific API errors here
+            print("OpenAI API error, please try again!")
+    
+
+pre_prompts = [
+    'Who are you?',
+    ('The history and development of Y6'
+       
+    ),
+    (
+        'Predict the LogP of Y6'
+    ),
+    'Generate a donor material with PCE = 10'
+]
+
+# sidebar
+with st.sidebar:
+
+    st.header("🤖 :blue[TeLLAgent] ")
+    # Input OpenAI api key
+    st.text_input(
+        'Input your OpenAI API key.',
+        placeholder = 'Input your OpenAI API key.',
+        type='password',
+        key='api_key',
+        on_change=on_api_key_change,
+        label_visibility="collapsed"
+    )
+ 
+    # Input model to use
+    st.selectbox(
+        'Select model to use',
+        ['gpt-4o-2024-11-20', 'deepseek-v3', 'gpt-4o-mini'],
+        key='model_select',
+    )
+
+    # Display prompt examples
+    st.markdown('# What can I ask?')
+    cols = st.columns(2)
+    with cols[0]:
+        st.button(
+           r'👑  Who     are    you  ?  🧨               ',
+            on_click=lambda: run_prompt(pre_prompts[0]),
+        )
+        st.button(
+            r'📚 The history and development of Y6     ',
+            on_click=lambda: run_prompt(pre_prompts[1]),
+        )
+    with cols[1]:
+        st.button(
+            r"🎄Predict the LogP of Y6  ",
+            on_click=lambda: run_prompt(pre_prompts[2]),
+        )
+        st.button(
+            r'💎 Generate a donor material with PCE = 10',
+            on_click=lambda: run_prompt(pre_prompts[3]),
+        )
+
+    st.selectbox(
+            'Select the file type ',
+            ['None', 'CSV (.csv)', 'Figure (.jpg, .png, .jpeg)', 'PDF (.pdf)'],
+            key='file_type',
+        )
+    uploaded_file = None
+    if ss.get('file_type') == 'Figure (.jpg, .png, .jpeg)':
+        uploaded_file = st.file_uploader("Choose a Figure", type = ["jpg", "jpeg", "png"])
+    if ss.get('file_type') == 'PDF (.pdf)':
+        uploaded_file = st.file_uploader("Choose a PDF file")
+    if ss.get('file_type') == 'CSV (.csv)':
+        uploaded_file = st.file_uploader("Choose a csv file", type = 'csv')   
+        
+    # Display available tools
+    st.markdown(f"# {len(tool_list)} available tools")
+    st.dataframe(
+        tool_list,
+        use_container_width=True,
+        hide_index=True,
+        height=200
+    )
+
+# Execute agent on user input
+if prompt := st.chat_input("Say something and/or attach files"):
+
+    if uploaded_file is not None:
+        if ss.get('file_type') == 'CSV (.csv)':
+            with tempfile.NamedTemporaryFile( dir = 'j:/', suffix ='.csv' ,delete=False) as f:
+                 f.write(uploaded_file.read())
+                 run_prompt(prompt + str(' ') +  str(f.name), file_path =  f.name) 
+                 f.close()
+      
+        if ss.get('file_type') == 'Figure (.jpg, .png, .jpeg)':
+    
+            st.image(uploaded_file, width = 500)
+            with tempfile.NamedTemporaryFile(dir = 'j:/',delete=False, suffix=".png") as temp:
+    
+                   mg_str = base64.b64encode(uploaded_file.getvalue()).decode("utf-8")
+                   temp.write(base64.b64decode(mg_str))
+             
+            run_prompt(prompt+ str(' ') + str(temp.name), image_path = temp.name )
+   
+        if ss.get('file_type') == 'PDF (.pdf)':
+                with tempfile.NamedTemporaryFile( dir = 'j:/', suffix ='.pdf' ,delete=False) as f:
+                     f.write(uploaded_file.read())
+                     run_prompt(prompt, file_path =  f.name) 
+                     f.close()
+      
+                    # with open("input.png","wb") as af:
+                    #              mg_str = base64.b64encode(files.getvalue()).decode("utf-8")
+                    #              af.write(base64.b64decode(mg_str))        
+                     
+                    # run_prompt(prompt.text+str(f.name), image_path =f.name )
+                # except:
+                #     st.markdown("Please input correct files or query ")
+    else:
+          run_prompt(prompt)

prompts.py

@@ -0,0 +1,110 @@
+# flake8: noqa
+PREFIX = """
+You are an AI system called TeLLAgent and your task is to respond to the question or
+solve the problem to the best of your ability using the provided tools.
+
+"""
+
+FORMAT_INSTRUCTIONS = """
+You can only respond with a single complete
+"Thought, Action, Action Input" format
+OR a single "Final Answer" format.
+
+Complete format:
+
+Thought: (reflect on your progress and decide what to do next)
+Action: (the action name, should be one of [{tool_names}])
+Action Input: (the input string to the action)
+
+OR
+
+Final Answer: (the final answer to the original input question)
+"""
+
+QUESTION_PROMPT1 = """
+Give detailed step-by-step solution to answer the question below using the following tools:
+Don't simplify the step description of the process.
+
+{tool_strings}
+
+Use the tools provided, using the most specific tool available for each action.
+Your final answer should contain all information necessary to answer the question and subquestions.
+
+IMPORTANT: Your first step is to check the following:    
+
+1. Does the question contain the name of the molecule, CAS, or molecular graph?
+   if so, as a first step, you should consider if it needs to convert the graph using graphconverter, name using Mol2SMILES or CAS number using Query2SMILES to SMILES.
+ 
+2. Were you asked to predict the power conversion efficiency (PCE) ? 
+    if so,  you are only allowed to choose one of the following tools.    
+    acceptor_predictor to predict the PCE of one acceptor molecule
+    donor_predictor to predict the PCE of one donor molecule
+    dap_predictor should be use when both the donor and acceptor molecule are offered
+
+3. Is the question about image,figure,graph or paper files ? 
+   if so, the papers or images have already been provided or referenced in some way.  
+   you should use ImageAnalysis or pdfreader  to solve the question.
+   Do not use other tools.
+   
+4. Were you ask to answer questions that require technical or general information , 
+   if so, you should combine the results from  WebSearch,  wikipedia and rag tool.
+    
+5. when you use the tool rag, you do not process the answer, return the results directly.   
+ 
+6.Do you need to work with images, you need to figure out the difference between the two tools Imageanalysis and graphconverter, 
+   if you want to get SMILES of molecules choose graphconverter, if you want to analyze or read images use Imageanalysis.
+
+Question: {input}
+"""
+
+QUESTION_PROMPT = """
+Answer the question below using the following tools:
+
+{tool_strings}
+
+Use the tools provided, using the most specific tool available for each action.
+Your final answer should contain all information necessary to answer the question and subquestions.
+
+IMPORTANT: Your first step is to check the following:    
+1. Were you need to convert the molecular graph, name or CAS number to SMILES.
+ if so, as a first step, you should use graphconvertor, Query2SMILES(chemspace_api_key), Query2CAS() tools.
+      
+2. Were you asked to predict the power conversion efficiency (PCE) ? 
+    if so,  as a first step, you should consider if it needs to convert the graph, name or CAS number to SMILES.
+    Then, as a second step, you are only allowed to choose one of the following tools.    
+    acceptor_predictor to predict the PCE of  acceptor molecule
+    donor_predictor to predict the PCE of donor molecule
+
+3. Were you ask to generate answer according to image or paper files,
+   if so, the papers or images have already been provided or referenced in some way.  
+   you are only allowed to output the answer by using ImageAnalysis and pdfreader  .
+   Do not use other tools.
+   
+4. Were you ask to answer questions that require technical or  general inquiry about a term or concept, 
+   if so, you should use rag tool first, then WebSearch, final wikipedia,
+   
+5. when you use the tool rag, you do not process the answer, return the results directly.
+Question: {input}
+"""
+
+SUFFIX = """
+
+Thought: {agent_scratchpad}
+"""
+FINAL_ANSWER_ACTION = "Final Answer:"
+
+
+REPHRASE_TEMPLATE = """In this exercise you will assume the role of a scientific assistant named TeLLAgent. Your task is to answer the provided question as best as you can, based on the provided solution draft.
+The solution draft follows the format "Thought, Action, Action Input, Observation", where the 'Thought' statements describe a reasoning sequence. The rest of the text is information obtained to complement the reasoning sequence, and it is 100% accurate.
+Your task is to write an answer to the question based on the solution draft, and the following guidelines:
+You need to be as detailed as possible in your answers to the questions and reduce the processing of the tool's output.
+Direct output the results when using Imageanalysis, codewriter, pdfreader, rag tools without further processing.
+
+The text should have an educative and assistant-like tone, be accurate, follow the same reasoning sequence than the solution draft and explain how any conclusion is reached.
+Question: {question}
+
+Solution draft: {agent_ans}
+
+Answer:
+"""
+ 

requirement.txt

@@ -0,0 +1,22 @@
+paper-scraper @ git+https://github.com/blackadad/paper-scraper.git
+numpy==1.26.4
+rdkit==2023.9.53
+streamlit
+google-search-results==2.4.2
+python-dotenv
+wikipedia
+torch==2.4.1 torchvision==0.19.1 torchaudio==2.4.1 --index-url https://download.pytorch.org/whl/cu118
+pydantic==2.10.4
+wandb
+langchain==0.3.19
+langchain-core==0.3.39
+langchain-community==0.3.18
+langchain-openai==0.3.1
+typing-inspect==0.8.0
+typing_extensions==4.12.2
+paper-qa
+pandas
+molsets
+scipy==1.15.2
+selfies==2.2.0
+browser-use

search.py

@@ -0,0 +1,32 @@
+from langchain_openai import ChatOpenAI
+from browser_use import Agent
+import asyncio
+from dotenv import load_dotenv
+load_dotenv()
+from langchain.tools import BaseTool
+
+class SMILES2Weight(BaseTool):
+    name: str = "webuse"
+    description: str = ("Calling the browser to search for information "
+                        "input query, return the searching results")
+
+    def __init__(
+        self,
+    ):
+        super().__init__()
+
+    async def _run(self, task: str) -> str:
+        agent = Agent(
+            task="Go to Reddit, search for 'browser-use', click on the first post and return the first comment.",
+            llm = ChatOpenAI(model="gpt-4o-2024-11-20",api_key='sk-itPrztYm9F6XZZpsBMJB9O7Vq0pYUABVVBSoThuBxEGTnDik',
+                 base_url="https://www.dmxapi.com/v1"),
+        )
+        result = await agent.run()
+        return result
+    
+    async def _arun(self, smiles: str) -> str:
+        """Use the tool asynchronously."""
+        raise NotImplementedError()
+        
+        
+ 

streamlit_callback_handler.py

@@ -0,0 +1,204 @@
+from typing import Any, Dict, List, Optional
+
+from langchain.callbacks.streamlit.streamlit_callback_handler import (
+
+    LLMThought,
+    LLMThoughtLabeler,
+    LLMThoughtState,
+    StreamlitCallbackHandler,
+    ToolRecord,
+)
+from langchain_core.agents import AgentAction, AgentFinish
+from streamlit.delta_generator import DeltaGenerator
+
+from utils import is_smiles
+
+import requests
+from langchain import LLMChain, PromptTemplate
+from langchain.chat_models import ChatOpenAI
+from rdkit import Chem
+
+
+def cdk(smiles):
+    """
+    Get a depiction of some smiles.
+    """
+
+    url = "https://www.simolecule.com/cdkdepict/depict/wob/svg"
+    headers = {"Content-Type": "application/json"}
+    response = requests.get(
+        url,
+        headers=headers,
+        params={
+            "smi": smiles,
+            "annotate": "colmap",
+            "zoom": 2,
+            "w": 150,
+            "h": 80,
+            "abbr": "off",
+        },
+    )
+    return response.text
+
+
+class LLMThoughtChem(LLMThought):
+    def __init__(
+        self,
+        parent_container: DeltaGenerator,
+        labeler: LLMThoughtLabeler,
+        expanded: bool,
+        collapse_on_complete: bool,
+    ):
+        super().__init__(
+            parent_container,
+            labeler,
+            expanded,
+            collapse_on_complete,
+        )
+
+    def on_tool_end(
+        self,
+        output: str,
+        color: Optional[str] = None,
+        observation_prefix: Optional[str] = None,
+        llm_prefix: Optional[str] = None,
+        output_ph: dict = {},
+        input_tool: str = "",
+        serialized: dict = {},
+        **kwargs: Any,
+    ) -> None:
+        # Depending on the tool name, decide what to display.
+        if serialized["name"] == "Name2SMILES":
+            safe_smiles = output.replace("[", "\[").replace("]", "\]")
+            if is_smiles(output):
+                self._container.markdown(
+                    f"**{safe_smiles}**{cdk(output)}", unsafe_allow_html=True
+                )
+
+        if serialized["name"] == "ReactionPredict":
+            rxn = f"{input_tool}>>{output}"
+            safe_smiles = rxn.replace("[", "\[").replace("]", "\]")
+            self._container.markdown(
+                f"**{safe_smiles}**{cdk(rxn)}", unsafe_allow_html=True
+            )
+
+        if serialized["name"] == "ReactionRetrosynthesis":
+            output = output.replace("[", "\[").replace("]", "\]")
+
+    def on_tool_start(
+        self, serialized: Dict[str, Any], input_str: str, **kwargs: Any
+    ) -> None:
+        # Called with the name of the tool we're about to run (in `serialized[name]`),
+        # and its input. We change our container's label to be the tool name.
+        self._state = LLMThoughtState.RUNNING_TOOL
+        tool_name = serialized["name"]
+        self._last_tool = ToolRecord(name=tool_name, input_str=input_str)
+        self._container.update(
+            new_label=(
+                self._labeler.get_tool_label(self._last_tool, is_complete=False)
+                .replace("[", "\[")
+                .replace("]", "\]")
+            )
+        )
+
+        # Display note of potential long time
+        if serialized["name"] == "ReactionRetrosynthesis" or serialized["name"] == "LiteratureSearch":
+            self._container.markdown(
+                f"‼️ Note: This tool can take some time to complete execution ‼️",
+                unsafe_allow_html=True,
+            )
+
+    def complete(self, final_label: Optional[str] = None) -> None:
+        """Finish the thought."""
+        if final_label is None and self._state == LLMThoughtState.RUNNING_TOOL:
+            assert (
+                self._last_tool is not None
+            ), "_last_tool should never be null when _state == RUNNING_TOOL"
+            final_label = self._labeler.get_tool_label(
+                self._last_tool, is_complete=True
+            )
+        self._state = LLMThoughtState.COMPLETE
+
+        final_label = final_label.replace("[", "\[").replace("]", "\]")
+        if self._collapse_on_complete:
+            self._container.update(new_label=final_label, new_expanded=False)
+        else:
+            self._container.update(new_label=final_label)
+
+
+class StreamlitCallbackHandlerChem(StreamlitCallbackHandler):
+    def __init__(
+        self,
+        parent_container: DeltaGenerator,
+        *,
+        max_thought_containers: int = 4,
+        expand_new_thoughts: bool = True,
+        collapse_completed_thoughts: bool = True,
+        thought_labeler: Optional[LLMThoughtLabeler] = None,
+        output_placeholder: dict = {},
+    ):
+        super(StreamlitCallbackHandlerChem, self).__init__(
+            parent_container,
+            max_thought_containers=max_thought_containers,
+            expand_new_thoughts=expand_new_thoughts,
+            collapse_completed_thoughts=collapse_completed_thoughts,
+            thought_labeler=thought_labeler,
+        )
+
+        self._output_placeholder = output_placeholder
+        self.last_input = ""
+
+    def on_llm_start(
+        self, serialized: Dict[str, Any], prompts: List[str], **kwargs: Any
+    ) -> None:
+        if self._current_thought is None:
+            self._current_thought = LLMThoughtChem(
+                parent_container=self._parent_container,
+                expanded=self._expand_new_thoughts,
+                collapse_on_complete=self._collapse_completed_thoughts,
+                labeler=self._thought_labeler,
+            )
+
+        self._current_thought.on_llm_start(serialized, prompts)
+
+        # We don't prune_old_thought_containers here, because our container won't
+        # be visible until it has a child.
+
+    def on_tool_start(
+        self, serialized: Dict[str, Any], input_str: str, **kwargs: Any
+    ) -> None:
+        self._require_current_thought().on_tool_start(serialized, input_str, **kwargs)
+        self._prune_old_thought_containers()
+        self._last_input = input_str
+        self._serialized = serialized
+
+    def on_tool_end(
+        self,
+        output: str,
+        color: Optional[str] = None,
+        observation_prefix: Optional[str] = None,
+        llm_prefix: Optional[str] = None,
+        **kwargs: Any,
+    ) -> None:
+        self._require_current_thought().on_tool_end(
+            output,
+            color,
+            observation_prefix,
+            llm_prefix,
+            output_ph=self._output_placeholder,
+            input_tool=self._last_input,
+            serialized=self._serialized,
+            **kwargs,
+        )
+        self._complete_current_thought()
+
+    def on_agent_finish(
+        self, finish: AgentFinish, color: Optional[str] = None, **kwargs: Any
+    ) -> None:
+        if self._current_thought is not None:
+            self._current_thought.complete(
+                self._thought_labeler.get_final_agent_thought_label()
+                .replace("[", "\[")
+                .replace("]", "\]")
+            )
+            self._current_thought = None

test.py

@@ -0,0 +1,262 @@
+# -*- coding: utf-8 -*-
+"""
+Created on Tue Mar 25 16:09:34 2025
+
+@author: BM109X32G-10GPU-02
+"""
+import os
+import sys
+import asyncio
+import streamlit as st
+from dotenv import load_dotenv
+
+# Ensure local repository (browser_use) is accessible
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from browser_use import Agent
+from browser_use.browser.browser import Browser, BrowserConfig
+from browser_use.controller.service import Controller
+from langchain_openai import ChatOpenAI
+# Load environment variables
+load_dotenv()
+
+if os.name == 'nt':
+    asyncio.set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy())
+    
+# Function to get the LLM based on provider
+def get_llm(provider: str):
+    if provider == 'anthropic':
+ 
+        api_key = 'sk-itPrztYm9F6XZZpsBMJB9O7Vq0pYUABVVBSoThuBxEGTnDik'
+        if not api_key:
+            st.error("Error: ANTHROPIC_API_KEY is not set. Please provide a valid API key.")
+            st.stop()
+        
+        return ChatOpenAI(openai_api_key = api_key, base_url="https://www.dmxapi.com/v1/")
+    elif provider == 'openai':
+        
+        api_key = 'sk-itPrztYm9F6XZZpsBMJB9O7Vq0pYUABVVBSoThuBxEGTnDik'
+        if not api_key:
+            st.error("Error: OPENAI_API_KEY is not set. Please provide a valid API key.")
+            st.stop()
+        
+        return ChatOpenAI(openai_api_key = api_key, base_url="https://www.dmxapi.com/v1/")
+    else:
+        st.error(f'Unsupported provider: {provider}')
+        st.stop()
+
+# Function to initialize the agent
+def initialize_agent(query: str, provider: str):
+    llm = get_llm(provider)
+    controller = Controller()
+    browser = Browser(config=BrowserConfig())
+
+    return Agent(
+        task=query,
+        llm=llm,
+        controller=controller,
+        browser=browser,
+        use_vision=True,
+        max_actions_per_step=1,
+    ), browser
+
+# Streamlit UI
+st.title("Automated Browser Agent with LLMs 🤖")
+if st.button("Foo"):
+    st.session_state.value = "Foo"
+    st.rerun()
+query = st.text_input("Enter your query:", "go to reddit and search for posts about browser-use")
+provider = st.radio("Select LLM Provider:", ["openai", "anthropic"], index=0)
+
+if st.button("Run Agent"):
+    st.write("Initializing agent...")
+    agent, browser = initialize_agent(query, provider)
+
+    async def run_agent():
+        with st.spinner("Running automation..."):
+            await agent.run(max_steps=25)
+        st.success("Task completed! 🎉")
+
+    asyncio.run(run_agent())
+
+    st.button("Close Browser", on_click=lambda: asyncio.run(browser.close()))
+import streamlit as st
+
+if "value" not in st.session_state:
+    st.session_state.value = "Title"
+
+##### Option using st.rerun #####
+st.header(st.session_state.value)
+
+
+# from langchain.chains import LLMChain, SimpleSequentialChain, RetrievalQA, ConversationalRetrievalChain
+
+# from langchain import PromptTemplate 
+ 
+# from langchain.tools import BaseTool
+ 
+# from langchain_core.messages import HumanMessage, SystemMessage
+# from langchain.base_language import BaseLanguageModel
+# from langchain.text_splitter import CharacterTextSplitter
+ 
+ 
+# from langchain_community.document_loaders import PyPDFLoader
+# from langchain_community.vectorstores import FAISS
+# from langchain_openai import ChatOpenAI
+# from langchain_openai import OpenAIEmbeddings
+
+# from langchain_community.document_loaders import PyPDFLoader
+# import streamlit as st
+# import pandas as pd
+# import base64
+
+# from langchain_community.embeddings import OllamaEmbeddings
+# from langchain.tools import BaseTool
+# from langchain_openai import ChatOpenAI
+# from langchain_core.messages import HumanMessage, SystemMessage
+# from langchain.base_language import BaseLanguageModel
+# import base64
+# from io import BytesIO
+# from PIL import Image
+# from langchain_community.embeddings import OllamaEmbeddings
+# from langchain.tools import BaseTool
+# from langchain_openai import ChatOpenAI
+# from langchain_core.messages import HumanMessage, SystemMessage
+# from langchain.base_language import BaseLanguageModel
+
+
+# from tempfile import NamedTemporaryFile
+# import streamlit as st
+
+# uploaded_file = st.file_uploader("File upload")
+
+# def convert_to_base64(pil_image):
+#     buffered = BytesIO()
+#     pil_image.save(buffered, format="PNG")
+#     img_str = base64.b64encode(buffered.getvalue()).decode("utf-8")
+#     return img_str
+
+# prompt = st.chat_input(
+#     "Say something and/or attach an image",
+#     accept_file=True,
+     
+# )
+# uploaded_file = st.file_uploader("Choose a file")
+# template = """
+
+#         You are an expert chemist and your task is to respond to the question or
+#         solve the problem to the best of your ability. You need to answer in as much detail as possible.
+#         You can only respond with a single "Final Answer" format.
+#         Use the following pieces of context to answer the question at the end. 
+#         If you don't know the answer, just say that you don't know, don't try to make up an answer.
+#         <context>
+#         {context}
+#         </context>
+
+#         Question: {question}
+#         Answer: 
+
+#         """
+        
+# import tempfile 
+# import base64
+# from io import BytesIO
+# from PIL import Image
+
+ 
+# if uploaded_file is  None:
+#     st.markdown(prompt)
+# if uploaded_file is not None:
+#            # try:
+ 
+#                      # loader = PyPDFLoader(uploaded_file)  
+#                      # documents = loader.load()  
+#                      # st.markdown(prompt)
+#             # except:    
+#             #     #try:
+ 
+#             #          file = pd.read_csv(uploaded_file)
+#             #          st.markdown(prompt)
+#    #  
+#    #  with tempfile.NamedTemporaryFile( dir = 'j:/', suffix ='.png' ,delete=False) as f:
+#          with tempfile.NamedTemporaryFile(dir = 'j:/',delete=False, suffix=".png") as temp_file:
+ 
+#                 mg_str = base64.b64encode(uploaded_file.getvalue()).decode("utf-8")
+ 
+ 
+#                 temp_file.write(base64.b64decode(mg_str))
+
+#          # image = Image.new('RGB', (100, 100), color='blue')
+#          # image.save(uploaded_file.getvalue(), format='PNG') 
+#          # loaded_image = Image.open(f.name)
+         
+#          pil_image = Image.open(temp_file.name)
+#          rgb_im = pil_image.convert('RGB')
+#          image_b64 = convert_to_base64(pil_image)
+#          query = 'what can you see in the image'
+#          llm = ChatOpenAI(model="gpt-4o-2024-11-20",api_key='sk-itPrztYm9F6XZZpsBMJB9O7Vq0pYUABVVBSoThuBxEGTnDik',
+#          base_url="https://www.dmxapi.com/v1")
+#          message = HumanMessage(
+#             content=[
+#                 {"type": "text", "text": query},
+#                 {
+#                     "type": "image_url",
+#                     "image_url": {"url":f"data:image/jpeg;base64,{image_b64}"},
+#                     },
+#                 ],)
+    
+#          response = llm.invoke([message])
+#          st.markdown(response.content) 
+         
+   # # mg_str = base64.b64encode(files.getvalue()).decode("utf-8")
+ 
+   # #  img_str = base64.b64encode(files.getvalue()).decode("utf-8")
+   # # # image_b64 = convert_to_base64(files.getvalue())
+
+   # #  
+   # #  st.markdown(query)
+   # #  st.markdown(response.content)
+    
+    
+    
+    
+
+         
+    
+    
+    
+    
+    
+    
+    
+    
+    
+    
+    
+    # with open("input.pdf","wb") as f:
+    #     base64_pdf = base64.b64encode(prompt["files"][0].read()).decode('utf-8')
+    #     f.write(base64.b64decode(base64_pdf))
+    # loader = PyPDFLoader(f.name)
+    # documents = loader.load()  
+    
+    # text_splitter = CharacterTextSplitter(chunk_size=6000, chunk_overlap=1000)
+    # docs = text_splitter.split_documents(documents) 
+    # embeddings =  OpenAIEmbeddings(model="text-embedding-3-large",api_key='sk-itPrztYm9F6XZZpsBMJB9O7Vq0pYUABVVBSoThuBxEGTnDik',
+    #      base_url="https://www.dmxapi.com/v1")
+
+   
+    # vectorstore = FAISS.from_documents(docs, embeddings)
+    # prompt = PromptTemplate(template=template, input_variables=[ "question"])
+    # qa_chain = RetrievalQA.from_chain_type(
+    #     llm=  ChatOpenAI(model="gpt-4o-2024-11-20",api_key='sk-itPrztYm9F6XZZpsBMJB9O7Vq0pYUABVVBSoThuBxEGTnDik',
+    #          base_url="https://www.dmxapi.com/v1"),
+    #     chain_type="stuff",
+    #     retriever=vectorstore.as_retriever(search_kwargs={"k": 2}),
+    #     return_source_documents=True,
+    #    chain_type_kwargs={"prompt": prompt},
+    # )
+     
+    # result = qa_chain.invoke('what is the main point')
+    # st.markdown(result['result'])
+    # a = pd.read_csv(prompt["files"][0] )
+    # st.markdown(a)

tools.py

@@ -0,0 +1,70 @@
+import os
+
+from langchain_community.agent_toolkits.load_tools import load_tools
+from langchain.base_language import BaseLanguageModel
+
+from tool import *
+
+
+def make_tools(llm: BaseLanguageModel, api_keys: dict = {}, verbose=True,  image_path = None, file_path = None):
+    serp_api_key = api_keys.get("SERP_API_KEY") or os.getenv("SERP_API_KEY")
+    image_path = image_path 
+    file_path = file_path
+    openai_api_key = api_keys.get("OPENAI_API_KEY") or os.getenv("OPENAI_API_KEY")
+    chemspace_api_key = api_keys.get("CHEMSPACE_API_KEY") or os.getenv(
+        "CHEMSPACE_API_KEY"
+    )
+    semantic_scholar_api_key = api_keys.get("SEMANTIC_SCHOLAR_API_KEY") or os.getenv(
+        "SEMANTIC_SCHOLAR_API_KEY"
+    )
+    serp_api_key = '3795acda6a74ea15033d34b54eac82982b26f559147d9cf04aca4bfca91c3e9d'
+    all_tools = load_tools(
+        [
+            #"python_repl",
+            # "ddg-search",
+            "wikipedia",
+            # "human"
+        ]
+    )
+
+    all_tools += [
+        browseruse(),
+        
+        rag(),
+        codewriter(),
+        
+        Query2SMILES(chemspace_api_key),
+        Mol2SMILES(chemspace_api_key) ,
+        Query2CAS(),
+        SMILES2Name(),
+        SMILES2SAScore(),
+        SMILES2LogP(),
+        SMILES2Properties(),
+        MolSimilarity(),
+        SMILES2Weight(),
+        FuncGroups(),
+        donor_predictor(),
+        acceptor_predictor(),
+        homolumo_predictor(),
+        dap_screen(),
+        graphconverter(),
+        molgen(),
+        dap_predictor(),
+        Scholar2ResultLLM(
+            llm=llm,
+            openai_api_key=openai_api_key,
+            semantic_scholar_api_key=semantic_scholar_api_key        ),
+    ]
+    
+    if serp_api_key:
+        all_tools += [WebSearch(serp_api_key)
+    ]
+    if image_path is not None:
+            all_tools += [Imageanalysis(image_path),
+                          
+        ]   
+    if file_path is not None:
+            all_tools += [pdfreader(file_path),
+        ]               
+ 
+    return all_tools

utils.py

@@ -0,0 +1,159 @@
+import re
+
+import requests
+from rdkit import Chem, DataStructs
+from rdkit.Chem import AllChem
+
+from urllib.request import urlopen
+from urllib.parse import quote
+
+def is_smiles(text):
+    try:
+        m = Chem.MolFromSmiles(text, sanitize=False)
+        if m is None:
+            return False
+        return True
+    except:
+        return False
+
+
+def is_multiple_smiles(text):
+    if is_smiles(text):
+        return "." in text
+    return False
+
+
+def split_smiles(text):
+    return text.split(".")
+
+
+def is_cas(text):
+    pattern = r"^\d{2,7}-\d{2}-\d$"
+    return re.match(pattern, text) is not None
+
+
+def largest_mol(smiles):
+    ss = smiles.split(".")
+    ss.sort(key=lambda a: len(a))
+    while not is_smiles(ss[-1]):
+        rm = ss[-1]
+        ss.remove(rm)
+    return ss[-1]
+
+
+def canonical_smiles(smiles):
+    try:
+        smi = Chem.MolToSmiles(Chem.MolFromSmiles(smiles), canonical=True)
+        return smi
+    except Exception:
+        return "Invalid SMILES string"
+
+
+def tanimoto(s1, s2):
+    """Calculate the Tanimoto similarity of two SMILES strings."""
+    try:
+        mol1 = Chem.MolFromSmiles(s1)
+        mol2 = Chem.MolFromSmiles(s2)
+        fp1 = AllChem.GetMorganFingerprintAsBitVect(mol1, 2, nBits=2048)
+        fp2 = AllChem.GetMorganFingerprintAsBitVect(mol2, 2, nBits=2048)
+        return DataStructs.TanimotoSimilarity(fp1, fp2)
+    except (TypeError, ValueError, AttributeError):
+        return "Error: Not a valid SMILES string"
+
+def CIRconvert(ids):
+   
+    url = 'http://cactus.nci.nih.gov/chemical/structure/' + quote(ids) + '/smiles'
+    ans = urlopen(url).read().decode('utf8')
+    return ans
+    
+    
+    
+def pubchem_query2smiles(
+    query: str,
+    url: str = "https://pubchem.ncbi.nlm.nih.gov/rest/pug/compound/name/{}/{}",
+) -> str:
+    if is_smiles(query):
+        if not is_multiple_smiles(query):
+            return query
+        else:
+            raise ValueError(
+                "Multiple SMILES strings detected, input one molecule at a time."
+            )
+    if url is None:
+        url = "https://pubchem.ncbi.nlm.nih.gov/rest/pug/compound/name/{}/{}"
+    r = requests.get(url.format(query, "property/IsomericSMILES/JSON"))
+    # convert the response to a json object
+    data = r.json()
+    # return the SMILES string
+    try:
+        smi = data["PropertyTable"]["Properties"][0]["IsomericSMILES"]
+    except: 
+        try: 
+            smi = CIRconvert(query)
+    
+        except KeyError:
+             return "Could not find a molecule matching the text. One possible cause is that the input is incorrect, input one molecule at a time."
+    return str(Chem.CanonSmiles(largest_mol(smi)))
+
+
+def query2cas(query: str, url_cid: str, url_data: str):
+    try:
+        mode = "name"
+        if is_smiles(query):
+            if is_multiple_smiles(query):
+                raise ValueError(
+                    "Multiple SMILES strings detected, input one molecule at a time."
+                )
+            mode = "smiles"
+        url_cid = url_cid.format(mode, query)
+        cid = requests.get(url_cid).json()["IdentifierList"]["CID"][0]
+        url_data = url_data.format(cid)
+        data = requests.get(url_data).json()
+    except (requests.exceptions.RequestException, KeyError):
+        raise ValueError("Invalid molecule input, no Pubchem entry")
+
+    try:
+        for section in data["Record"]["Section"]:
+            if section.get("TOCHeading") == "Names and Identifiers":
+                for subsection in section["Section"]:
+                    if subsection.get("TOCHeading") == "Other Identifiers":
+                        for subsubsection in subsection["Section"]:
+                            if subsubsection.get("TOCHeading") == "CAS":
+                                return subsubsection["Information"][0]["Value"][
+                                    "StringWithMarkup"
+                                ][0]["String"]
+    except KeyError:
+        raise ValueError("Invalid molecule input, no Pubchem entry")
+
+    raise ValueError("CAS number not found")
+
+
+def smiles2name(smi, single_name=True):
+    """This function queries the given molecule smiles and returns a name record or iupac"""
+
+    try:
+        smi = Chem.MolToSmiles(Chem.MolFromSmiles(smi), canonical=True)
+    except Exception:
+        raise ValueError("Invalid SMILES string")
+    # query the PubChem database
+    r = requests.get(
+        "https://pubchem.ncbi.nlm.nih.gov/rest/pug/compound/smiles/"
+        + smi
+        + "/synonyms/JSON"
+    )
+    # convert the response to a json object
+    data = r.json()
+    # return the SMILES string
+    try:
+        if single_name:
+            index = 0
+            names = data["InformationList"]["Information"][0]["Synonym"]
+            while is_cas(name := names[index]):
+                index += 1
+                if index == len(names):
+                    raise ValueError("No name found")
+        else:
+            name = data["InformationList"]["Information"][0]["Synonym"]
+    except KeyError:
+        raise ValueError("Unknown Molecule")
+    return name