"""Implements Python Code Generation. """
from __future__ import annotations

import re
from typing import Any, Dict, List, Optional

from bs4 import BeautifulSoup
from langchain.base_language import BaseLanguageModel
from langchain.callbacks.manager import CallbackManagerForChainRun
from langchain.chains.base import Chain
from langchain.prompts.base import BasePromptTemplate
from langchain.prompts.chat import (
    ChatPromptTemplate,
    HumanMessagePromptTemplate,
)
from langchain.schema import SystemMessage
from loguru import logger
from pydantic import BaseModel, Extra

from real_agents.adapters.data_model import MessageDataModel
from real_agents.adapters.memory import ReadOnlySharedStringMemory
from real_agents.data_agent.evaluation.python_evaluator import PythonEvaluator
from real_agents.data_agent.python.echarts_prompt import E_SYSTEM_PROMPT, ECHARTS_REF_CODE, ECHARTS_USER_PROMPT
from real_agents.data_agent.python.system_prompt import SYSTEM_PROMPT
from real_agents.data_agent.python.python_prompt import USER_PROMPT
from real_agents.adapters.llm import LLMChain

class PythonChain(Chain, BaseModel):
    """Chain for Generating Python Code"""

    llm_chain: LLMChain

    memory: Optional[ReadOnlySharedStringMemory] = None
    stop: str = "\n\n"
    get_answer_expr: str = ""
    python_globals: Optional[Dict[str, Any]] = None
    python_locals: Optional[Dict[str, Any]] = None
    output_key: str = "result"  #: :meta private:
    return_intermediate_steps: bool = False
    code_execution_mode: str = "local"
    jupyter_kernel_pool: Optional[Any] = None
    reference_code: str = ""

    chat_id: Optional[str] = None
    user_id: Optional[str] = None

    class Config:
        """Configuration for this pydantic object."""

        extra = Extra.ignore
        arbitrary_types_allowed = True

    def _validate_inputs(self, inputs: Dict[str, str]) -> None:
        """Check that all inputs are present."""
        missing_keys = set(self.input_keys).difference(inputs)
        if "chat_history" in missing_keys:
            missing_keys.remove("chat_history")
        if missing_keys:
            raise ValueError(f"Missing some input keys: {missing_keys}")

    @property
    def input_keys(self) -> List[str]:
        """Return the singular input key.

        :meta private:
        """
        return ["data_info", "question", "chat_history"]

    @property
    def output_keys(self) -> List[str]:
        """Return the singular output key.

        :meta private:
        """
        if not self.return_intermediate_steps:
            return [self.output_key]
        else:
            return [self.output_key, "intermediate_steps"]

    def _call(
        self,
        inputs: Dict[str, str],
        run_manager: Optional[CallbackManagerForChainRun] = None,
    ) -> Dict[str, str]:
        logger.bind(msg_head="PythonChain inputs").trace(inputs)

        _run_manager = run_manager or CallbackManagerForChainRun.get_noop_manager()
        _run_manager.on_text(inputs[self.input_keys[0]])
        inputs["chat_history"] = ""
        if self.memory is not None:
            inputs["chat_history"] = self.memory.load_memory_variables({})["chat_history"]
            inputs["chat_history"] = MessageDataModel.extract_code_for_python_tool(inputs["chat_history"])

        history = {
            "history_code": inputs["chat_history"],
            "question": inputs["question"],
            "data": inputs["data_info"],
            "reference_code": self.reference_code,
        }

        # we apply llm as a magic function, which serves as python code generation func.
        raw_output = self.llm_chain.run(**history)

        def _extract_code(_raw_output: str) -> str:
            # Using 'html.parser' to parse the content
            soup = BeautifulSoup(_raw_output, "html.parser")
            try:
                _raw_output = soup.find("code").text
            except:
                pass
            if "```python:" in _raw_output:
                pattern = r"```python\n{(.*?)}\n```"
                match = re.search(pattern, _raw_output, re.DOTALL)
                if match:
                    return match.group(1)
                else:
                    return _raw_output
            else:
                return _raw_output

        code = _extract_code(raw_output).replace("\\n", "\n")

        logger.bind(msg_head="PythonChain generated program").trace(code)

        repl = PythonEvaluator(
            code_execution_mode=self.code_execution_mode,
            jupyter_kernel_pool=self.jupyter_kernel_pool,
        )

        """
        Since there will be error if we try to launch matplotlib GUI in the server,
        I add this line to avoid backend execution of matplotlib for now.
        """
        result = repl.run(code + f"\n{self.get_answer_expr}", user_id=self.user_id, chat_id=self.chat_id)

        logger.bind(msg_head="PythonChain execution result").trace(result)

        output = {self.output_key: result}
        if self.return_intermediate_steps:
            output["intermediate_steps"] = code
        return output

    @classmethod
    def create_python_prompt(cls, system_prompt: str, reference_code_prompt: str) -> BasePromptTemplate:
        input_variables = ["history_code", "question", "data", "reference_code"]
        messages = [
            SystemMessage(content=system_prompt),
            HumanMessagePromptTemplate.from_template(template=USER_PROMPT),
        ]

        return ChatPromptTemplate(input_variables=input_variables, messages=messages)

    @classmethod
    def create_echarts_prompt(cls, system_prompt: str, reference_code_prompt: str) -> BasePromptTemplate:
        input_variables = ["history_code", "question", "data", "reference_code"]
        messages = [
            SystemMessage(content=system_prompt),
            HumanMessagePromptTemplate.from_template(template=ECHARTS_USER_PROMPT),
        ]

        return ChatPromptTemplate(input_variables=input_variables, messages=messages)

    @classmethod
    def from_python_prompt(cls, llm: BaseLanguageModel, **kwargs: Any) -> PythonChain:
        """Load from Echarts prompt."""
        llm_chain = LLMChain(llm=llm, prompt=cls.create_python_prompt(SYSTEM_PROMPT, ""))
        return cls(
            llm_chain=llm_chain,
            get_answer_expr="",
            reference_code="",
            **kwargs,
        )

    @classmethod
    def from_echarts_prompt(cls, llm: BaseLanguageModel, **kwargs: Any) -> PythonChain:
        """Load from Echarts prompt."""
        llm_chain = LLMChain(llm=llm, prompt=cls.create_echarts_prompt(E_SYSTEM_PROMPT, ""))
        return cls(
            llm_chain=llm_chain,
            get_answer_expr="",
            reference_code=ECHARTS_REF_CODE,
            **kwargs,
        )

    @property
    def _chain_type(self) -> str:
        return "program_chain"


PythonChain

from langchain.callbacks.manager import CallbackManagerForChainRun
from langchain.chains.base import Chain
from langchain.prompts.base import BasePromptTemplate
from langchain.prompts.chat import (
    ChatPromptTemplate,
    HumanMessagePromptTemplate,
)
from langchain.schema import SystemMessage
from loguru import logger
from pydantic import BaseModel, Extra

from real_agents.adapters.data_model import MessageDataModel
from real_agents.adapters.memory import ReadOnlySharedStringMemory
from real_agents.data_agent.evaluation.python_evaluator import PythonEvaluator
from real_agents.data_agent.python.echarts_prompt import E_SYSTEM_PROMPT, ECHARTS_REF_CODE, ECHARTS_USER_PROMPT
from real_agents.data_agent.python.system_prompt import SYSTEM_PROMPT
from real_agents.data_agent.python.python_prompt import USER_PROMPT
from real_agents.adapters.llm import LLMChain

class PythonChain(Chain, BaseModel):
    """Chain for Generating Python Code"""

    llm_chain: LLMChain

    memory: Optional[ReadOnlySharedStringMemory] = None
    stop: str = "\n\n"
    get_answer_expr: str = ""
    python_globals: Optional[Dict[str, Any]] = None
    python_locals: Optional[Dict[str, Any]] = None
    output_key: str = "result"  #: :meta private:
    return_intermediate_steps: bool = False
    code_execution_mode: str = "local"
    jupyter_kernel_pool: Optional[Any] = None
    reference_code: str = ""

    chat_id: Optional[str] = None
    user_id: Optional[str] = None

    class Config:
        """Configuration for this pydantic object."""

        extra = Extra.ignore
        arbitrary_types_allowed = True

    def _validate_inputs(self, inputs: Dict[str, str]) -> None:
        """Check that all inputs are present."""
        missing_keys = set(self.input_keys).difference(inputs)
        if "chat_history" in missing_keys:
            missing_keys.remove("chat_history")
        if missing_keys:
            raise ValueError(f"Missing some input keys: {missing_keys}")

    @property
    def input_keys(self) -> List[str]:
        """Return the singular input key.

        :meta private:
        """
        return ["data_info", "question", "chat_history"]

    @property
    def output_keys(self) -> List[str]:
        """Return the singular output key.

        :meta private:
        """
        if not self.return_intermediate_steps:
            return [self.output_key]
        else:
            return [self.output_key, "intermediate_steps"]

    def _call(
        self,
        inputs: Dict[str, str],
        run_manager: Optional[CallbackManagerForChainRun] = None,
    ) -> Dict[str, str]:
        logger.bind(msg_head="PythonChain inputs").trace(inputs)

        _run_manager = run_manager or CallbackManagerForChainRun.get_noop_manager()
        _run_manager.on_text(inputs[self.input_keys[0]])
        inputs["chat_history"] = ""
        if self.memory is not None:
            inputs["chat_history"] = self.memory.load_memory_variables({})["chat_history"]
            inputs["chat_history"] = MessageDataModel.extract_code_for_python_tool(inputs["chat_history"])

        history = {
            "history_code": inputs["chat_history"],
            "question": inputs["question"],
            "data": inputs["data_info"],
            "reference_code": self.reference_code,
        }

        # we apply llm as a magic function, which serves as python code generation func.
        raw_output = self.llm_chain.run(**history)

        def _extract_code(_raw_output: str) -> str:
            # Using 'html.parser' to parse the content
            soup = BeautifulSoup(_raw_output, "html.parser")
            try:
                _raw_output = soup.find("code").text
            except:
                pass
            if "```python:" in _raw_output:
                pattern = r"```python\n{(.*?)}\n```"
                match = re.search(pattern, _raw_output, re.DOTALL)
                if match:
                    return match.group(1)
                else:
                    return _raw_output
            else:
                return _raw_output

        code = _extract_code(raw_output).replace("\\n", "\n")

        logger.bind(msg_head="PythonChain generated program").trace(code)

        repl = PythonEvaluator(
            code_execution_mode=self.code_execution_mode,
            jupyter_kernel_pool=self.jupyter_kernel_pool,
        )

        """
        Since there will be error if we try to launch matplotlib GUI in the server,
        I add this line to avoid backend execution of matplotlib for now.
        """
        result = repl.run(code + f"\n{self.get_answer_expr}", user_id=self.user_id, chat_id=self.chat_id)

        logger.bind(msg_head="PythonChain execution result").trace(result)

        output = {self.output_key: result}
        if self.return_intermediate_steps:
            output["intermediate_steps"] = code
        return output

    @classmethod
    def create_python_prompt(cls, system_prompt: str, reference_code_prompt: str) -> BasePromptTemplate:
        input_variables = ["history_code", "question", "data", "reference_code"]
        messages = [
            SystemMessage(content=system_prompt),
            HumanMessagePromptTemplate.from_template(template=USER_PROMPT),
        ]

        return ChatPromptTemplate(input_variables=input_variables, messages=messages)

    @classmethod
    def create_echarts_prompt(cls, system_prompt: str, reference_code_prompt: str) -> BasePromptTemplate:
        input_variables = ["history_code", "question", "data", "reference_code"]
        messages = [
            SystemMessage(content=system_prompt),
            HumanMessagePromptTemplate.from_template(template=ECHARTS_USER_PROMPT),
        ]

        return ChatPromptTemplate(input_variables=input_variables, messages=messages)

    @classmethod
    def from_python_prompt(cls, llm: BaseLanguageModel, **kwargs: Any) -> PythonChain:
        """Load from Echarts prompt."""
        llm_chain = LLMChain(llm=llm, prompt=cls.create_python_prompt(SYSTEM_PROMPT, ""))
        return cls(
            llm_chain=llm_chain,
            get_answer_expr="",
            reference_code="",
            **kwargs,
        )

    @classmethod
    def from_echarts_prompt(cls, llm: BaseLanguageModel, **kwargs: Any) -> PythonChain:
        """Load from Echarts prompt."""
        llm_chain = LLMChain(llm=llm, prompt=cls.create_echarts_prompt(E_SYSTEM_PROMPT, ""))
        return cls(
            llm_chain=llm_chain,
            get_answer_expr="",
            reference_code=ECHARTS_REF_CODE,
            **kwargs,
        )

    @property
    def _chain_type(self) -> str:
        return "program_chain"


Config

class PythonChain(Chain, BaseModel):
    """Chain for Generating Python Code"""

    llm_chain: LLMChain

    memory: Optional[ReadOnlySharedStringMemory] = None
    stop: str = "\n\n"
    get_answer_expr: str = ""
    python_globals: Optional[Dict[str, Any]] = None
    python_locals: Optional[Dict[str, Any]] = None
    output_key: str = "result"  #: :meta private:
    return_intermediate_steps: bool = False
    code_execution_mode: str = "local"
    jupyter_kernel_pool: Optional[Any] = None
    reference_code: str = ""

    chat_id: Optional[str] = None
    user_id: Optional[str] = None

    class Config:
        """Configuration for this pydantic object."""

        extra = Extra.ignore
        arbitrary_types_allowed = True

    def _validate_inputs(self, inputs: Dict[str, str]) -> None:
        """Check that all inputs are present."""
        missing_keys = set(self.input_keys).difference(inputs)
        if "chat_history" in missing_keys:
            missing_keys.remove("chat_history")
        if missing_keys:
            raise ValueError(f"Missing some input keys: {missing_keys}")

    @property
    def input_keys(self) -> List[str]:
        """Return the singular input key.

        :meta private:
        """
        return ["data_info", "question", "chat_history"]

    @property
    def output_keys(self) -> List[str]:
        """Return the singular output key.


    memory: Optional[ReadOnlySharedStringMemory] = None
    stop: str = "\n\n"
    get_answer_expr: str = ""
    python_globals: Optional[Dict[str, Any]] = None
    python_locals: Optional[Dict[str, Any]] = None
    output_key: str = "result"  #: :meta private:
    return_intermediate_steps: bool = False
    code_execution_mode: str = "local"
    jupyter_kernel_pool: Optional[Any] = None
    reference_code: str = ""

    chat_id: Optional[str] = None
    user_id: Optional[str] = None

    class Config:
        """Configuration for this pydantic object."""

        extra = Extra.ignore
        arbitrary_types_allowed = True

    def _validate_inputs(self, inputs: Dict[str, str]) -> None:
        """Check that all inputs are present."""
        missing_keys = set(self.input_keys).difference(inputs)
        if "chat_history" in missing_keys:
            missing_keys.remove("chat_history")
        if missing_keys:
            raise ValueError(f"Missing some input keys: {missing_keys}")

    @property
    def input_keys(self) -> List[str]:
        """Return the singular input key.

        :meta private:
        """
        return ["data_info", "question", "chat_history"]

    @property
    def output_keys(self) -> List[str]:
        """Return the singular output key.

        :meta private:
        """
        if not self.return_intermediate_steps:
            return [self.output_key]
        else:
            return [self.output_key, "intermediate_steps"]


    reference_code: str = ""

    chat_id: Optional[str] = None
    user_id: Optional[str] = None

    class Config:
        """Configuration for this pydantic object."""

        extra = Extra.ignore
        arbitrary_types_allowed = True

    def _validate_inputs(self, inputs: Dict[str, str]) -> None:
        """Check that all inputs are present."""
        missing_keys = set(self.input_keys).difference(inputs)
        if "chat_history" in missing_keys:
            missing_keys.remove("chat_history")
        if missing_keys:
            raise ValueError(f"Missing some input keys: {missing_keys}")

    @property
    def input_keys(self) -> List[str]:
        """Return the singular input key.

        :meta private:
        """
        return ["data_info", "question", "chat_history"]

    @property
    def output_keys(self) -> List[str]:
        """Return the singular output key.

        :meta private:
        """
        if not self.return_intermediate_steps:
            return [self.output_key]
        else:
            return [self.output_key, "intermediate_steps"]

    def _call(
        self,
        inputs: Dict[str, str],
        run_manager: Optional[CallbackManagerForChainRun] = None,
    ) -> Dict[str, str]:
        logger.bind(msg_head="PythonChain inputs").trace(inputs)

        _run_manager = run_manager or CallbackManagerForChainRun.get_noop_manager()


        extra = Extra.ignore
        arbitrary_types_allowed = True

    def _validate_inputs(self, inputs: Dict[str, str]) -> None:
        """Check that all inputs are present."""
        missing_keys = set(self.input_keys).difference(inputs)
        if "chat_history" in missing_keys:
            missing_keys.remove("chat_history")
        if missing_keys:
            raise ValueError(f"Missing some input keys: {missing_keys}")

    @property
    def input_keys(self) -> List[str]:
        """Return the singular input key.

        :meta private:
        """
        return ["data_info", "question", "chat_history"]

    @property
    def output_keys(self) -> List[str]:
        """Return the singular output key.

        :meta private:
        """
        if not self.return_intermediate_steps:
            return [self.output_key]
        else:
            return [self.output_key, "intermediate_steps"]

    def _call(
        self,
        inputs: Dict[str, str],
        run_manager: Optional[CallbackManagerForChainRun] = None,
    ) -> Dict[str, str]:
        logger.bind(msg_head="PythonChain inputs").trace(inputs)

        _run_manager = run_manager or CallbackManagerForChainRun.get_noop_manager()
        _run_manager.on_text(inputs[self.input_keys[0]])
        inputs["chat_history"] = ""
        if self.memory is not None:
            inputs["chat_history"] = self.memory.load_memory_variables({})["chat_history"]
            inputs["chat_history"] = MessageDataModel.extract_code_for_python_tool(inputs["chat_history"])

        history = {
            "history_code": inputs["chat_history"],
            "question": inputs["question"],
            "data": inputs["data_info"],
            "reference_code": self.reference_code,


    @property
    def input_keys(self) -> List[str]:
        """Return the singular input key.

        :meta private:
        """
        return ["data_info", "question", "chat_history"]

    @property
    def output_keys(self) -> List[str]:
        """Return the singular output key.

        :meta private:
        """
        if not self.return_intermediate_steps:
            return [self.output_key]
        else:
            return [self.output_key, "intermediate_steps"]

    def _call(
        self,
        inputs: Dict[str, str],
        run_manager: Optional[CallbackManagerForChainRun] = None,
    ) -> Dict[str, str]:
        logger.bind(msg_head="PythonChain inputs").trace(inputs)

        _run_manager = run_manager or CallbackManagerForChainRun.get_noop_manager()
        _run_manager.on_text(inputs[self.input_keys[0]])
        inputs["chat_history"] = ""
        if self.memory is not None:
            inputs["chat_history"] = self.memory.load_memory_variables({})["chat_history"]
            inputs["chat_history"] = MessageDataModel.extract_code_for_python_tool(inputs["chat_history"])

        history = {
            "history_code": inputs["chat_history"],
            "question": inputs["question"],
            "data": inputs["data_info"],
            "reference_code": self.reference_code,
        }

        # we apply llm as a magic function, which serves as python code generation func.
        raw_output = self.llm_chain.run(**history)

        def _extract_code(_raw_output: str) -> str:
            # Using 'html.parser' to parse the content
            soup = BeautifulSoup(_raw_output, "html.parser")
            try:
                _raw_output = soup.find("code").text
            except:
                pass
            if "```python:" in _raw_output:
                pattern = r"```python\n{(.*?)}\n```"
                match = re.search(pattern, _raw_output, re.DOTALL)
                if match:
                    return match.group(1)
                else:
                    return _raw_output
            else:
                return _raw_output

        code = _extract_code(raw_output).replace("\\n", "\n")

        logger.bind(msg_head="PythonChain generated program").trace(code)

        repl = PythonEvaluator(
            code_execution_mode=self.code_execution_mode,
            jupyter_kernel_pool=self.jupyter_kernel_pool,
        )

        """
        Since there will be error if we try to launch matplotlib GUI in the server,
        I add this line to avoid backend execution of matplotlib for now.
        """
        result = repl.run(code + f"\n{self.get_answer_expr}", user_id=self.user_id, chat_id=self.chat_id)

        logger.bind(msg_head="PythonChain execution result").trace(result)

        output = {self.output_key: result}
        if self.return_intermediate_steps:
            output["intermediate_steps"] = code
        return output

    @classmethod
    def create_python_prompt(cls, system_prompt: str, reference_code_prompt: str) -> BasePromptTemplate:
        input_variables = ["history_code", "question", "data", "reference_code"]
        messages = [
            SystemMessage(content=system_prompt),
            HumanMessagePromptTemplate.from_template(template=USER_PROMPT),
        ]

        return ChatPromptTemplate(input_variables=input_variables, messages=messages)

    @classmethod
    def create_echarts_prompt(cls, system_prompt: str, reference_code_prompt: str) -> BasePromptTemplate:
        input_variables = ["history_code", "question", "data", "reference_code"]
        messages = [
            SystemMessage(content=system_prompt),
            HumanMessagePromptTemplate.from_template(template=ECHARTS_USER_PROMPT),
        ]

        return ChatPromptTemplate(input_variables=input_variables, messages=messages)


        repl = PythonEvaluator(
            code_execution_mode=self.code_execution_mode,
            jupyter_kernel_pool=self.jupyter_kernel_pool,
        )

        """
        Since there will be error if we try to launch matplotlib GUI in the server,
        I add this line to avoid backend execution of matplotlib for now.
        """
        result = repl.run(code + f"\n{self.get_answer_expr}", user_id=self.user_id, chat_id=self.chat_id)

        logger.bind(msg_head="PythonChain execution result").trace(result)

        output = {self.output_key: result}
        if self.return_intermediate_steps:
            output["intermediate_steps"] = code
        return output

    @classmethod
    def create_python_prompt(cls, system_prompt: str, reference_code_prompt: str) -> BasePromptTemplate:
        input_variables = ["history_code", "question", "data", "reference_code"]
        messages = [
            SystemMessage(content=system_prompt),
            HumanMessagePromptTemplate.from_template(template=USER_PROMPT),
        ]

        return ChatPromptTemplate(input_variables=input_variables, messages=messages)

    @classmethod
    def create_echarts_prompt(cls, system_prompt: str, reference_code_prompt: str) -> BasePromptTemplate:
        input_variables = ["history_code", "question", "data", "reference_code"]
        messages = [
            SystemMessage(content=system_prompt),
            HumanMessagePromptTemplate.from_template(template=ECHARTS_USER_PROMPT),
        ]

        return ChatPromptTemplate(input_variables=input_variables, messages=messages)

    @classmethod
    def from_python_prompt(cls, llm: BaseLanguageModel, **kwargs: Any) -> PythonChain:
        """Load from Echarts prompt."""
        llm_chain = LLMChain(llm=llm, prompt=cls.create_python_prompt(SYSTEM_PROMPT, ""))
        return cls(
            llm_chain=llm_chain,
            get_answer_expr="",
            reference_code="",
            **kwargs,


        result = repl.run(code + f"\n{self.get_answer_expr}", user_id=self.user_id, chat_id=self.chat_id)

        logger.bind(msg_head="PythonChain execution result").trace(result)

        output = {self.output_key: result}
        if self.return_intermediate_steps:
            output["intermediate_steps"] = code
        return output

    @classmethod
    def create_python_prompt(cls, system_prompt: str, reference_code_prompt: str) -> BasePromptTemplate:
        input_variables = ["history_code", "question", "data", "reference_code"]
        messages = [
            SystemMessage(content=system_prompt),
            HumanMessagePromptTemplate.from_template(template=USER_PROMPT),
        ]

        return ChatPromptTemplate(input_variables=input_variables, messages=messages)

    @classmethod
    def create_echarts_prompt(cls, system_prompt: str, reference_code_prompt: str) -> BasePromptTemplate:
        input_variables = ["history_code", "question", "data", "reference_code"]
        messages = [
            SystemMessage(content=system_prompt),
            HumanMessagePromptTemplate.from_template(template=ECHARTS_USER_PROMPT),
        ]

        return ChatPromptTemplate(input_variables=input_variables, messages=messages)

    @classmethod
    def from_python_prompt(cls, llm: BaseLanguageModel, **kwargs: Any) -> PythonChain:
        """Load from Echarts prompt."""
        llm_chain = LLMChain(llm=llm, prompt=cls.create_python_prompt(SYSTEM_PROMPT, ""))
        return cls(
            llm_chain=llm_chain,
            get_answer_expr="",
            reference_code="",
            **kwargs,
        )

    @classmethod
    def from_echarts_prompt(cls, llm: BaseLanguageModel, **kwargs: Any) -> PythonChain:
        """Load from Echarts prompt."""
        llm_chain = LLMChain(llm=llm, prompt=cls.create_echarts_prompt(E_SYSTEM_PROMPT, ""))
        return cls(
            llm_chain=llm_chain,
            get_answer_expr="",
            reference_code=ECHARTS_REF_CODE,


    def create_python_prompt(cls, system_prompt: str, reference_code_prompt: str) -> BasePromptTemplate:
        input_variables = ["history_code", "question", "data", "reference_code"]
        messages = [
            SystemMessage(content=system_prompt),
            HumanMessagePromptTemplate.from_template(template=USER_PROMPT),
        ]

        return ChatPromptTemplate(input_variables=input_variables, messages=messages)

    @classmethod
    def create_echarts_prompt(cls, system_prompt: str, reference_code_prompt: str) -> BasePromptTemplate:
        input_variables = ["history_code", "question", "data", "reference_code"]
        messages = [
            SystemMessage(content=system_prompt),
            HumanMessagePromptTemplate.from_template(template=ECHARTS_USER_PROMPT),
        ]

        return ChatPromptTemplate(input_variables=input_variables, messages=messages)

    @classmethod
    def from_python_prompt(cls, llm: BaseLanguageModel, **kwargs: Any) -> PythonChain:
        """Load from Echarts prompt."""
        llm_chain = LLMChain(llm=llm, prompt=cls.create_python_prompt(SYSTEM_PROMPT, ""))
        return cls(
            llm_chain=llm_chain,
            get_answer_expr="",
            reference_code="",
            **kwargs,
        )

    @classmethod
    def from_echarts_prompt(cls, llm: BaseLanguageModel, **kwargs: Any) -> PythonChain:
        """Load from Echarts prompt."""
        llm_chain = LLMChain(llm=llm, prompt=cls.create_echarts_prompt(E_SYSTEM_PROMPT, ""))
        return cls(
            llm_chain=llm_chain,
            get_answer_expr="",
            reference_code=ECHARTS_REF_CODE,
            **kwargs,
        )

    @property
    def _chain_type(self) -> str:
        return "program_chain"


        input_variables = ["history_code", "question", "data", "reference_code"]
        messages = [
            SystemMessage(content=system_prompt),
            HumanMessagePromptTemplate.from_template(template=ECHARTS_USER_PROMPT),
        ]

        return ChatPromptTemplate(input_variables=input_variables, messages=messages)

    @classmethod
    def from_python_prompt(cls, llm: BaseLanguageModel, **kwargs: Any) -> PythonChain:
        """Load from Echarts prompt."""
        llm_chain = LLMChain(llm=llm, prompt=cls.create_python_prompt(SYSTEM_PROMPT, ""))
        return cls(
            llm_chain=llm_chain,
            get_answer_expr="",
            reference_code="",
            **kwargs,
        )

    @classmethod
    def from_echarts_prompt(cls, llm: BaseLanguageModel, **kwargs: Any) -> PythonChain:
        """Load from Echarts prompt."""
        llm_chain = LLMChain(llm=llm, prompt=cls.create_echarts_prompt(E_SYSTEM_PROMPT, ""))
        return cls(
            llm_chain=llm_chain,
            get_answer_expr="",
            reference_code=ECHARTS_REF_CODE,
            **kwargs,
        )

    @property
    def _chain_type(self) -> str:
        return "program_chain"


        llm_chain = LLMChain(llm=llm, prompt=cls.create_python_prompt(SYSTEM_PROMPT, ""))
        return cls(
            llm_chain=llm_chain,
            get_answer_expr="",
            reference_code="",
            **kwargs,
        )

    @classmethod
    def from_echarts_prompt(cls, llm: BaseLanguageModel, **kwargs: Any) -> PythonChain:
        """Load from Echarts prompt."""
        llm_chain = LLMChain(llm=llm, prompt=cls.create_echarts_prompt(E_SYSTEM_PROMPT, ""))
        return cls(
            llm_chain=llm_chain,
            get_answer_expr="",
            reference_code=ECHARTS_REF_CODE,
            **kwargs,
        )

    @property
    def _chain_type(self) -> str:
        return "program_chain"


    ) -> Dict[str, str]:
        logger.bind(msg_head="PythonChain inputs").trace(inputs)

        _run_manager = run_manager or CallbackManagerForChainRun.get_noop_manager()
        _run_manager.on_text(inputs[self.input_keys[0]])
        inputs["chat_history"] = ""
        if self.memory is not None:
            inputs["chat_history"] = self.memory.load_memory_variables({})["chat_history"]
            inputs["chat_history"] = MessageDataModel.extract_code_for_python_tool(inputs["chat_history"])

        history = {
            "history_code": inputs["chat_history"],
            "question": inputs["question"],
            "data": inputs["data_info"],
            "reference_code": self.reference_code,
        }

        # we apply llm as a magic function, which serves as python code generation func.
        raw_output = self.llm_chain.run(**history)

        def _extract_code(_raw_output: str) -> str:
            # Using 'html.parser' to parse the content
            soup = BeautifulSoup(_raw_output, "html.parser")
            try:
                _raw_output = soup.find("code").text
            except:
                pass
            if "```python:" in _raw_output:
                pattern = r"```python\n{(.*?)}\n```"
                match = re.search(pattern, _raw_output, re.DOTALL)
                if match:
                    return match.group(1)
                else:
                    return _raw_output
            else:
                return _raw_output

        code = _extract_code(raw_output).replace("\\n", "\n")

        logger.bind(msg_head="PythonChain generated program").trace(code)

        repl = PythonEvaluator(
            code_execution_mode=self.code_execution_mode,
            jupyter_kernel_pool=self.jupyter_kernel_pool,
        )

        """
        Since there will be error if we try to launch matplotlib GUI in the server,
        I add this line to avoid backend execution of matplotlib for now.
        """
        result = repl.run(code + f"\n{self.get_answer_expr}", user_id=self.user_id, chat_id=self.chat_id)

        logger.bind(msg_head="PythonChain execution result").trace(result)

        output = {self.output_key: result}
        if self.return_intermediate_steps:


ECHARTS_REF_CODE = """Here are some examples of generating Py-Echarts Code based on the given table(s). Please generate new one based on the data and question human asks you, import the neccessary libraries and make sure the code is correct.

IMPORTANT: You need to follow the coding style, and the type of the x, y axis. But also need to focus on the column name of the uploaded tables(if exists). Generally, PyEcharts does not accept numpy.int or numpy.float, etc. It only supports built-in data type like int, float, and str.

Given the following database:
company_sales.xlsx
   year  sales  profit  expenses  employees
0  2010    100      60        40         10
1  2011    120      80        50         12
2  2012    150      90        60         14
3  2013    170     120        70         16
[too long to show]

Q: Could you help plot a bar chart with the year on the x-axis and the sales on the y-axis?

import pandas as pd
from pyecharts.charts import Bar
from pyecharts import options as opts
df = pd.read_excel('company_sales.xlsx')
years = [str(_) for _ in df['year'].tolist()]
sales = [float(_) for _ in df['sales'].tolist()]
bar = Bar()
bar.add_xaxis(years)
bar.add_yaxis("Sales", sales)
bar.set_global_opts(
    xaxis_opts=opts.AxisOpts(
        type_="category",
        name="Year",
    ),
    yaxis_opts=opts.AxisOpts(
        type_="value",
        name="Sales",
    ),
    title_opts=opts.TitleOpts(title="Sales over Years"),
)
# Render the chart
ret_json = bar.dump_options()
print(ret_json)

Given the same `company_sales.xlsx`.
Q: A line chart comparing sales and profit over time would be useful. Could you help plot it?

import pandas as pd
from pyecharts.charts import Line
from pyecharts import options as opts
df = pd.read_excel('company_sales.xlsx')
year = [str(_) for _ in df["year"].to_list()]
sales = [float(_) for _ in df["sales"].to_list()]
profit = [float(_) for _ in df["profit"].to_list()]
line = Line()
# Add x-axis and y-axis data
line.add_xaxis(year)
line.add_yaxis("Sales", sales)
line.add_yaxis("Profit", profit)
line.set_global_opts(
    xaxis_opts=opts.AxisOpts(
        type_="category", # better use category rather than value
        name="year",
        min_=min(year),
        max_=max(year),
    ),
    yaxis_opts=opts.AxisOpts(
        type_="value",
        name="price",
    ),
    title_opts=opts.TitleOpts(title="Sales and Profit over Time"),
)
ret_json = line.dump_options()
print(ret_json)

Given the same `company_sales.xlsx`.
Q: A `stacked` line chart comparing sales and profit over time would be useful. Could you help plot it?
Note: stacked line chart is more fancy in display, while the former is more neat.

import pandas as pd
from pyecharts.charts import Line
from pyecharts import options as opts
df = pd.read_excel('company_sales.xlsx')
year = [str(_) for _ in df["year"].to_list()] # better use category rather than value
sales = [float(_) for _ in df["sales"].to_list()]
profit = [float(_) for _ in df["year"].to_list()]
line = Line()
# Add x-axis and y-axis data
line.add_xaxis(year)
line.add_yaxis("Sales", df["sales"].tolist(), stack="")
line.add_yaxis("Profit", df["profit"].tolist(), stack="")
line.set_global_opts(
    xaxis_opts=opts.AxisOpts(
        type_="category",
        name="year",
        min_=min(year),
        max_=max(year),
    ),
    yaxis_opts=opts.AxisOpts(
        type_="value",
        name="price",
        axistick_opts=opts.AxisTickOpts(is_show=True),
        splitline_opts=opts.SplitLineOpts(is_show=True),
    ),
    title_opts=opts.TitleOpts(title="Sales and Profit over Time"),
)
line.set_series_opts(
    areastyle_opts=opts.AreaStyleOpts(opacity=0.5),
)
ret_json = line.dump_options()
print(ret_json)

Given the following database:
shop_sales.tsv
   shop_id  total_sales  espresso_sales  latte_sales  cappuccino_sales  city_population
0        1         5000            1500         2000              1500           500000
1        2         5500            1800         2200              1500           800000
2        3         6000            2000         2500              1500          1200000
3        4         4500            1300         1800              1400           300000
4        5         6200            2200         2700              1300           600000
Q: I would like a pie chart showing the sales proportion of espresso, latte, and cappuccino for Shop 1.

import pandas as pd
from pyecharts.charts import Pie
from pyecharts import options as opts
df = pd.read_csv('shop_sales.tsv', sep='\\t')
shop1 = df.loc[df['shop_id'] == 1]
data_pair = [
    ('Espresso', float(shop1['espresso_sales'].item())), # pair must be (str, int/float)
    ('Latte', float(shop1['latte_sales'].item())),       # pair must be (str, int/float)
    ('Cappuccino', int(shop1['cappuccino_sales'].item())), # pair must be (str, int/float)
]
pie = Pie()
pie.add(
    series_name="Sales Breakdown",
    data_pair=data_pair,
    radius=["30%", "75%"],
)
pie.set_global_opts(
    title_opts=opts.TitleOpts(
        title="Coffee Sales Breakdown for Shop 1",
    ),
)
ret_json = pie.dump_options()
print(ret_json)

Q: Generate a scatter plot.

import random
from pyecharts import options as opts
from pyecharts.charts import Scatter
from pyecharts.faker import Faker

# Create some random data
data = [(random.randint(0, 100), random.randint(0, 100)) for _ in range(10)]
x = [i[0] for i in data]
y = [i[1] for i in data]
print(data)
scatter = Scatter()
scatter.add_xaxis(x)
scatter.add_yaxis("size", y)
scatter.set_global_opts(
    xaxis_opts=opts.AxisOpts(type_="value"), # scatter x axis must be numeric
    yaxis_opts=opts.AxisOpts(type_="value"), # scatter y axis must be numeric
    title_opts=opts.TitleOpts(title="Scatter Plot Example"),
    visualmap_opts=opts.VisualMapOpts(type_="size", max_=max(y), min_=min(y)),
)
ret_json = scatter.dump_options()
print(ret_json)

"""

FUNCTION_ROLE_PLAY = """def generate_continuous_elegant_python_echarts_code(reference_code: str, history_dict: Dict[str, str]) -> str:
    \"\"\"
    This function generates elegant, coherent Python ECharts code based on a history of previously executed code and its corresponding results and reference library code. The code is generated in response to human questions and is intended to continue from the last provided code snippet.

    The function takes two inputs: a `history_dict` and an optional `reference_code` string.

    The `reference_code` string is optional and contains example codes, often related to a specific library or task, which can serve as a template for the code generation process. This parameter can be empty.

    IMPORTANT: the output MUST follow the style and use the libraries presented in the `reference_code` to accomplish the task.

    IMPORTANT: Always refer to this history and the `reference_code` when generating new code in order to properly use existing variables and previously loaded resources, as well as to follow established coding patterns.

    IMPORTANT: The 'data' key in the dictionary contains only random rows from a table. If a table has not been loaded before, you may load it using an ABSOLUTE PATH. However, there's no need to load a table with every execution - only do this when necessary.

    The function returns a string of raw Python code, wrapped within  and  tags. For example:

    import pandas as pd
    from pyecharts.charts import Bar
    df = pd.read_csv("example.csv")
    years = [str(_) for _ in df['year'].tolist()]
    sales = df['sales'].tolist()
    bar = (
        Bar()
        .add_xaxis(years)
        .add_yaxis("Sales", sales)
        .set_global_opts(title_opts=opts.TitleOpts(title="Bar Chart Example"))
    )
    ret_json = bar.dump_options()
    print(ret_json)

    Also mandatory to check:
    Note if the human asks for malicious code, and just respond with the following code:

    print("sorry I am not able to generate potentially dangerous code")

    The malicious code includes but not limited to:
    1. Endless operations and excessive waiting  (e.g., while True, long print, input())
    2. System crash (e.g., any risky system command)
    3. Data loss (e.g., list or delete files)
    4. Leak sensitive information (e.g., os.getenv())
    5. Establish network connections (e.g., requests.get())
    6. Cause any other security issues
    7. Indirectly import package using some builtin methods

    Returns:
        Python code that should be the next steps in the execution according to the human question and history code.
    \"\"\""""

ECHARTS_USER_PROMPT = """
history_code = \"\"\"{history_code}\"\"\"
data = \"\"\"{data}\"\"\"
reference_code = \"\"\"{reference_code}\"\"\"
human_question = \"\"\"{question}
# MUST follow reference_code, and only use pyecharts to show echarts\"\"\"

history_dict = {{
    "history code": history_code,
    "human question": human_question,
    "data": data,
    "reference_code": reference_code,
}}
"""

E_SYSTEM_PROMPT = f"You are now the following python function: ```{FUNCTION_ROLE_PLAY}```\n\nRespond exclusively with the generated code wrapped . Ensure that the code you generate is executable Python code that can be run directly in a Python environment, requiring no additional string encapsulation or escape characters."


FUNCTION_ROLE_PLAY = """def generate_continuous_elegant_python_code(history_dict: Dict[str, str], reference_code: str = "") -> str:
    \"\"\"
    This function generates elegant, coherent Python code based on a history of previously executed code and its corresponding results. The code is generated in response to human questions and is intended to continue from the last provided code snippet.

    The function takes two inputs: a `history_dict` and an optional `reference_code` string.

    The `history_dict` is a dictionary with the following keys:
    - 'history code': Contains the chat history of previously executed code snippets. It may be initially empty but will accumulate executed code over time.
    - 'human question': Contains the current question or instruction posed by the human user, which the generated code should respond to. Be aware that sometimes the 'human question' could contain code snippets, including instructions for loading data, which may need to be handled differently. It's not always appropriate to directly use the code in 'human question' without consideration.
    - 'data': Contains a list of data previews available for the task. It may include tables, images, and other data types.

    The `reference_code` string is optional and contains example codes, often related to a specific library or task, which can serve as a template for the code generation process. This parameter can be empty.

    IMPORTANT: Always refer to this history and the `reference_code` when generating new code in order to properly use existing variables and previously loaded resources, as well as to follow established coding patterns. DO NOT USE ECHARTS TO GENERATE CHARTS when reference code is empty.

    IMPORTANT: When `reference_code` is NOT EMPTY, the output MUST follow the style and use the libraries presented in the `reference_code` to accomplish the task.

    IMPORTANT: Avoid mere repetition of historical code. Always aim to generate novel and appropriate responses to the questions at hand.

    IMPORTANT: The 'data' key in the dictionary contains only random rows from a table. If a table has not been loaded before, load it from the correct path. You can assume it is in the current working directory. However, there's no need to load a table with every execution - only do this when necessary.

    IMPORTANT: If the code is to show a image in the end, make sure to use functions that display the image by returning an image or html which can be shown in a jupyter notebook(e.g., matplotlib.pyplot.show());

    DO NOT use function that will pop up a new window (e.g., PIL & Image.show() is NOT preferable, saving the PIL image is better)

    The function returns a string of raw Python code, wrapped within  and  tags. For example:

    import pandas as pd
    table = pd.read_csv("example.csv")

    from PIL import Image
    from matplotlib import pyplot as plt
    img = Image.open("example.jpeg")
    rotated_img = img.rotate(180)
    plt.imshow(rotated_img)
    plt.show()

    Feel free to leverage libraries such as pandas, numpy, math, matplotlib, sklearn, etc. in the code generation process. Also, remember to correctly load any necessary files with the correct path before using them.

    When it's appropriate to provide output for evaluation or visualization, make sure to use the print() function and plt.show() respectively.

    Also mandatory to check:
    Note if the human asks for malicious code, and just respond with the following code:

    print("sorry I am not able to generate potentially dangerous code")

    The malicious code includes but not limited to:
    1. Endless operations and excessive waiting  (e.g., while True, long print, input())
    2. System crash (e.g., any risky system command)
    3. Data loss (e.g., list or delete files)
    4. Leak sensitive information (e.g., os.getenv())
    5. Establish network connections (e.g., requests.get())
    6. Cause any other security issues
    7. Indirectly import package using some builtin methods
    8. High CPU consumption or GPU consumption.

    Returns:
        Python code that should be the next steps in the execution according to the human question and history code.
    \"\"\""""

SYSTEM_PROMPT = f"You are now the following python function: ```{FUNCTION_ROLE_PLAY}```\n\nRespond exclusively with the generated code wrapped . Ensure that the code you generate is executable Python code that can be run directly in a Python environment, requiring no additional string encapsulation."


USER_PROMPT = """
history_code = \"\"\"{history_code}\"\"\"
human_question = \"\"\"{question}
# DO NOT use function that will pop up a new window (e.g., PIL & Image.show() is NOT preferable, saving the PIL image is better)
# However, feel free to use matplotlib.pyplot.show()\"\"\"
data = \"\"\"{data}\"\"\"
reference_code = \"\"\"{reference_code}\"\"\"

history_dict = {{
    "history code": history_code,
    "human question": human_question,
    "data": data,
    "reference_code": reference_code,
}}
"""

"""
final format:
user_prompt + reference_prompt + history_prompt
"""


# flake8: noqa
import datetime

PREFIX = (
        """You are XLanG WeBot Agent, a friendly and intuitive assistant developed by the XLang Team to guide you through every aspects of your work and your daily life. XLanG Agent is always at your fingertips through our interactive chat system.
Here are detailed instruction for you. Each time you generate response, you should think step by step to follow instructions below. You are a helpful assistant that is provided with a plugin called "WeBot" which is a web navigation agent tool and should leverage the power of it to help human to fulfill their needs, such as booking a hotel, buying a ticket, or searching for information, etc.
Human will ask you questions, and you can use WeBot to help them, they are assumed to know nothing about the WeBot.
----------------------------
Here are something you MUST remember:
1. After receiving output from the WeBot, you should check
    1.1 whether WeBot was interrupted, if so you should NEVER try again by yourself.
    1.2 whether WeBot failed or had error(not because of interruption), if so you should tell the human the error.
2. Today is
""".strip() + " "
        + datetime.datetime.now().strftime("%Y-%m-%d")
        + """, and you should adapt the input to fit into the date, for example, seasonal information, or today's date as coordinate, etc.

NEVER EVER EVER use other plugins except WeBot.
TRY YOUR BEST to break the question down into several parts and answer them one by one.
TRY YOUR BEST to use the WeBot to help you answer the question, you don't need to mention that you will use which WeBot, just use it.

To make your response informative, always speak includes the following information in MARKDOWN format when responding a message, that is:
1. Natural language explanation, that make explain the API output in a human readable way;
2. Organized information such as bullet points or MARKDOWN tables, followed by the links to the items (that in the API output), news etc. if API output contains the information;
3. The links should in MARKDOWN format and have value in it. If reference information is provided in the API output, like links to the items, news etc. Your explanation MUST provide the links on each items and links can be clicked on when API output contains the information. The links better attach on some natural language explanation through MARKDOWN syntax, for example, - [Renewable Energy - Center for Climate and Energy Solutions](https://www.c2es.org/content/renewable-energy/);
4. If there are image we would like to display, please use MARKDOWN syntax to display it, for example, ![image](https://www.c2es.org/content/renewable-energy/);
5. Try to speak more and show all the information you got in a organized way, that will make you a better assistant, especially when you are giving the final answer.

PLUGINS
------
The plugins you can use are:
""".strip() + "\n"
)

FORMAT_INSTRUCTIONS = """RESPONSE FORMAT INSTRUCTIONS
----------------------------

When you use tools or generate final answer, please output a response in one of two formats:
**Option 1: Explain and Use WeBot**
If the response involves using a WeBot, you can start with a natural language explanation[Optional], plus exactly one WeBot calling[MUST], and ends with no more words. The WeBot calling format should be a markdown code snippet with the following JSON schema:

```json
{{{{
    "action": string wrapped with \"\", // The action to take. Must be WeBot
    "action_input": string wrapped with \"\" // Natural language query to be input to the WeBot.
}}}}
```
NEVER EVER EVER make up a plugin except [{tool_names}]
NEVER EVER EVER generate code as action input when using WeBot. Just input natural language by using/paraphrasing human query.
(Please note that ONLY ONE WeBot should be used per response.)

**Option #2:**
If you want to respond directly to the human without using a WeBot, provide a plain natural language response. However, if you initially generated a natural language response and then decide to use a WeBot, make sure to include the WeBot action and input after the initial response.

Begin.
"""

SUFFIX = "{input}"

TEMPLATE_TOOL_RESPONSE = """PLUGINS RESPONSE:
---------------------
{observation}

THOUGHT
--------------------

Okay, So what's next? Are the WeBot's response enough to answer human's initial query? Please follow these instructions:

1. Evaluate WeBot Response [Mandatory]: Carefully evaluate the WeBot's response and determine if it sufficiently addresses the human's query. Consider the content and implications of the WeBot's response.

```json
{{{{
    "action": string wrapped with \"\", // The action to take. Must be one of WeBot
    "action_input": string wrapped with \"\" // Natural language query to be input to the WeBot
}}}}
```
(Please note that ONLY ONE WeBot should be used per response.)

3. Deliver Comprehensive Answer [Optional 2 or 3]: If the WeBot response sufficiently addresses the query, deliver a comprehensive answer to the human. Focus solely on the content and implications of the WeBot's response. MUST NOT include explanations of the WeBot's functions.

Note. you must do 1; For 2 and 3, You must choose one from them.

Begin.
"""

# models like anthropic claude-v1 or claude-2 can only return valid completion with human message as the last message, so we append the fake AI message at the end.
fake_continue_prompt = {
    "claude-2": "you can start to think and respond to me using the above formats. No Apology. Just respond with format in Option 2(use tool) or Option 3(direct text response), no other words.\n\nBegin.",
    "claude-v1": "you can start to think and respond to me using the above formats. No Apology. Just respond with format in Option 2(use tool) or Option 3(direct text response), no other words.\n\nBegin.",
}


"""An agent designed to hold a conversation in addition to using tools. (Specially designed for plugins model)"""
from __future__ import annotations

from typing import Any, List, Optional, Sequence, Tuple, Union
from pydantic import Extra, Field
from typing_extensions import override

from langchain.agents.agent import AgentOutputParser
from langchain.agents.utils import validate_tools_single_input
from langchain.base_language import BaseLanguageModel
from langchain.chains import LLMChain
from langchain.schema import (
    AgentAction,
    AgentFinish,
    AIMessage,
    BaseMessage,
    BaseOutputParser,
    HumanMessage
)

from langchain.callbacks.manager import (
    Callbacks
)
from langchain.tools.base import BaseTool
from langchain.prompts import (
    BasePromptTemplate,
    ChatPromptTemplate,
    HumanMessagePromptTemplate,
    MessagesPlaceholder,
    SystemMessagePromptTemplate,
)

from real_agents.adapters.agent_helpers.agent import Agent
from real_agents.adapters.agent_helpers.output_parser import ConversationOutputParser
from real_agents.web_agent.webot_prompt import (
    PREFIX,
    SUFFIX,
    TEMPLATE_TOOL_RESPONSE,
    fake_continue_prompt
)
from real_agents.adapters.data_model import DataModel, MessageDataModel
from real_agents.data_agent.copilot import ConversationalChatAgent

class ConversationalWebotChatAgent(ConversationalChatAgent):  # fixme: change it to Agent will leads to bug, but why?
    """An agent designed to hold a conversation in addition to using plugin tool."""

    output_parser: ConversationOutputParser = Field(
        default_factory=ConversationOutputParser()
    )

    template_tool_response: str = TEMPLATE_TOOL_RESPONSE
    continue_model: Optional[str] = None

    class Config:
        """Configuration for this pydantic object."""

        extra = Extra.allow
        arbitrary_types_allowed = True

    @classmethod
    def _get_default_output_parser(
            cls, **kwargs: Any
    ) -> ConversationOutputParser:
        return ConversationOutputParser()

    @property
    def _agent_type(self) -> str:
        raise NotImplementedError

    @property
    def observation_prefix(self) -> str:
        """Prefix to append the observation with."""
        return "Observation: "

    @property
    def llm_prefix(self) -> str:
        """Prefix to append the llm call with."""
        return "Thought:"

    @classmethod
    def _validate_tools(cls, tools: Sequence[BaseTool]) -> None:
        super()._validate_tools(tools)
        validate_tools_single_input(cls.__name__, tools)

    @classmethod
    def create_prompt(
            cls,
            tools: Sequence[BaseTool],
            system_message: str = PREFIX,
            human_message: str = SUFFIX,
            input_variables: Optional[List[str]] = None,
            output_parser: Optional[BaseOutputParser] = None,
    ) -> BasePromptTemplate:
        tool_strings = "\n".join([f"Name: {tool.name}\nDescription: {tool.description}" for tool in tools])

        tool_names = ", ".join([tool.name for tool in tools])
        _output_parser = output_parser or cls._get_default_output_parser()

        format_instructions = _output_parser.get_format_instructions("webot")
        format_instructions = format_instructions.format(tool_names=tool_names)

        # system message
        system_message = system_message + f"{tool_strings}\n\n{format_instructions}"

        # human input
        final_prompt = human_message

        if input_variables is None:
            input_variables = ["input", "chat_history", "agent_scratchpad"]
        messages = [
            SystemMessagePromptTemplate.from_template(system_message),
            MessagesPlaceholder(variable_name="chat_history"),
            HumanMessagePromptTemplate.from_template(final_prompt),
            MessagesPlaceholder(variable_name="agent_scratchpad"),
        ]
        return ChatPromptTemplate(input_variables=input_variables, messages=messages)

    def _construct_scratchpad(self, intermediate_steps: List[Tuple[AgentAction, str]]) -> List[BaseMessage]:
        """Construct the scratchpad that lets the agent continue its thought process."""
        thoughts: List[BaseMessage] = []
        # Try to only use AI message for scratchpad
        content = []
        for idx, (action, full_observation) in enumerate(intermediate_steps):
            content.append(MessageDataModel.extract_action_for_llm(action.log))

            observation = full_observation
            if isinstance(full_observation, DataModel):
                llm_raw_observation = full_observation.get_llm_side_data()

                observation = MessageDataModel.extract_tool_response_for_llm(llm_raw_observation, tool_style="plugin")

            if idx == len(intermediate_steps) - 1:
                tool_response = self.template_tool_response.format(
                    observation=str(observation), tool_names=self.allowed_tools
                )
                if idx == len(intermediate_steps) - 1:
                    content.append(tool_response)
                else:
                    content.append(observation)

        content_str = "\n".join(content)
        thoughts.append(AIMessage(content=content_str))
        if self.continue_model is not None and len(intermediate_steps) != 0:
            thoughts.append(HumanMessage(content=fake_continue_prompt[self.continue_model]))
        return thoughts

    @override
    def plan(
            self,
            intermediate_steps: List[Tuple[AgentAction, str]],
            **kwargs: Any,
    ) -> Union[AgentAction, AgentFinish]:
        """Given input, decided what to do.

        Args:
            intermediate_steps: Steps the LLM has taken to date,
                along with observations
            **kwargs: User inputs.

        Returns:
            Action specifying what tool to use.
        """
        full_inputs = self.get_full_inputs(intermediate_steps, **kwargs)
        system_prompt = self.llm_chain.prompt.messages[0].format().content
        system_prompt_tokens = MessageDataModel._count_tokens(
            system_prompt
        )
        max_tokens = 8000
        max_gen_tokens = 1000
        # FIXME: need more accurate token limit calculation
        full_inputs = MessageDataModel.truncate_chat_history(full_inputs, max_token=max_tokens - system_prompt_tokens - max_gen_tokens)
        full_output = self.llm_chain.predict(**full_inputs)

        return self.output_parser.parse(full_output)

    @classmethod
    def from_llm_and_tools(
            cls,
            llm: BaseLanguageModel,
            tools: Sequence[BaseTool],
            callbacks: Callbacks = None,
            output_parser: Optional[AgentOutputParser] = None,
            system_message: str = PREFIX,
            human_message: str = SUFFIX,
            input_variables: Optional[List[str]] = None,
            **kwargs: Any,
    ) -> Agent:
        """Construct an agent from an LLM and tools."""
        cls._validate_tools(tools)

        _output_parser = output_parser or cls._get_default_output_parser()
        prompt = cls.create_prompt(
            tools,
            system_message=system_message,
            human_message=human_message,
            input_variables=input_variables,
            output_parser=_output_parser,
        )
        llm_chain = LLMChain(
            llm=llm,
            prompt=prompt,
        )
        tool_names = [tool.name for tool in tools]
        return cls(
            llm_chain=llm_chain,
            allowed_tools=tool_names,
            output_parser=_output_parser,
            **kwargs,
        )


ConversationalWebotChatAgent

from langchain.prompts import (
    BasePromptTemplate,
    ChatPromptTemplate,
    HumanMessagePromptTemplate,
    MessagesPlaceholder,
    SystemMessagePromptTemplate,
)

from real_agents.adapters.agent_helpers.agent import Agent
from real_agents.adapters.agent_helpers.output_parser import ConversationOutputParser
from real_agents.web_agent.webot_prompt import (
    PREFIX,
    SUFFIX,
    TEMPLATE_TOOL_RESPONSE,
    fake_continue_prompt
)
from real_agents.adapters.data_model import DataModel, MessageDataModel
from real_agents.data_agent.copilot import ConversationalChatAgent

class ConversationalWebotChatAgent(ConversationalChatAgent):  # fixme: change it to Agent will leads to bug, but why?
    """An agent designed to hold a conversation in addition to using plugin tool."""

    output_parser: ConversationOutputParser = Field(
        default_factory=ConversationOutputParser()
    )

    template_tool_response: str = TEMPLATE_TOOL_RESPONSE
    continue_model: Optional[str] = None

    class Config:
        """Configuration for this pydantic object."""

        extra = Extra.allow
        arbitrary_types_allowed = True

    @classmethod
    def _get_default_output_parser(
            cls, **kwargs: Any
    ) -> ConversationOutputParser:
        return ConversationOutputParser()

    @property
    def _agent_type(self) -> str:
        raise NotImplementedError

    @property
    def observation_prefix(self) -> str:
        """Prefix to append the observation with."""
        return "Observation: "

    @property
    def llm_prefix(self) -> str:
        """Prefix to append the llm call with."""
        return "Thought:"

    @classmethod
    def _validate_tools(cls, tools: Sequence[BaseTool]) -> None:
        super()._validate_tools(tools)
        validate_tools_single_input(cls.__name__, tools)

    @classmethod
    def create_prompt(
            cls,
            tools: Sequence[BaseTool],
            system_message: str = PREFIX,
            human_message: str = SUFFIX,
            input_variables: Optional[List[str]] = None,
            output_parser: Optional[BaseOutputParser] = None,
    ) -> BasePromptTemplate:
        tool_strings = "\n".join([f"Name: {tool.name}\nDescription: {tool.description}" for tool in tools])

        tool_names = ", ".join([tool.name for tool in tools])
        _output_parser = output_parser or cls._get_default_output_parser()

        format_instructions = _output_parser.get_format_instructions("webot")
        format_instructions = format_instructions.format(tool_names=tool_names)

        # system message
        system_message = system_message + f"{tool_strings}\n\n{format_instructions}"

        # human input
        final_prompt = human_message

        if input_variables is None:
            input_variables = ["input", "chat_history", "agent_scratchpad"]
        messages = [
            SystemMessagePromptTemplate.from_template(system_message),
            MessagesPlaceholder(variable_name="chat_history"),
            HumanMessagePromptTemplate.from_template(final_prompt),
            MessagesPlaceholder(variable_name="agent_scratchpad"),
        ]
        return ChatPromptTemplate(input_variables=input_variables, messages=messages)

    def _construct_scratchpad(self, intermediate_steps: List[Tuple[AgentAction, str]]) -> List[BaseMessage]:
        """Construct the scratchpad that lets the agent continue its thought process."""
        thoughts: List[BaseMessage] = []
        # Try to only use AI message for scratchpad
        content = []
        for idx, (action, full_observation) in enumerate(intermediate_steps):
            content.append(MessageDataModel.extract_action_for_llm(action.log))

            observation = full_observation
            if isinstance(full_observation, DataModel):
                llm_raw_observation = full_observation.get_llm_side_data()

                observation = MessageDataModel.extract_tool_response_for_llm(llm_raw_observation, tool_style="plugin")

            if idx == len(intermediate_steps) - 1:
                tool_response = self.template_tool_response.format(
                    observation=str(observation), tool_names=self.allowed_tools
                )
                if idx == len(intermediate_steps) - 1:
                    content.append(tool_response)
                else:
                    content.append(observation)

        content_str = "\n".join(content)
        thoughts.append(AIMessage(content=content_str))
        if self.continue_model is not None and len(intermediate_steps) != 0:
            thoughts.append(HumanMessage(content=fake_continue_prompt[self.continue_model]))
        return thoughts

    @override
    def plan(
            self,
            intermediate_steps: List[Tuple[AgentAction, str]],
            **kwargs: Any,
    ) -> Union[AgentAction, AgentFinish]:
        """Given input, decided what to do.

        Args:
            intermediate_steps: Steps the LLM has taken to date,
                along with observations
            **kwargs: User inputs.

        Returns:
            Action specifying what tool to use.
        """
        full_inputs = self.get_full_inputs(intermediate_steps, **kwargs)
        system_prompt = self.llm_chain.prompt.messages[0].format().content
        system_prompt_tokens = MessageDataModel._count_tokens(
            system_prompt
        )
        max_tokens = 8000
        max_gen_tokens = 1000
        # FIXME: need more accurate token limit calculation
        full_inputs = MessageDataModel.truncate_chat_history(full_inputs, max_token=max_tokens - system_prompt_tokens - max_gen_tokens)
        full_output = self.llm_chain.predict(**full_inputs)

        return self.output_parser.parse(full_output)

    @classmethod
    def from_llm_and_tools(
            cls,
            llm: BaseLanguageModel,
            tools: Sequence[BaseTool],
            callbacks: Callbacks = None,
            output_parser: Optional[AgentOutputParser] = None,
            system_message: str = PREFIX,
            human_message: str = SUFFIX,
            input_variables: Optional[List[str]] = None,
            **kwargs: Any,
    ) -> Agent:
        """Construct an agent from an LLM and tools."""
        cls._validate_tools(tools)

        _output_parser = output_parser or cls._get_default_output_parser()
        prompt = cls.create_prompt(
            tools,
            system_message=system_message,
            human_message=human_message,
            input_variables=input_variables,
            output_parser=_output_parser,
        )
        llm_chain = LLMChain(
            llm=llm,
            prompt=prompt,
        )
        tool_names = [tool.name for tool in tools]
        return cls(
            llm_chain=llm_chain,
            allowed_tools=tool_names,
            output_parser=_output_parser,
            **kwargs,
        )


Config

from real_agents.web_agent.webot_prompt import (
    PREFIX,
    SUFFIX,
    TEMPLATE_TOOL_RESPONSE,
    fake_continue_prompt
)
from real_agents.adapters.data_model import DataModel, MessageDataModel
from real_agents.data_agent.copilot import ConversationalChatAgent

class ConversationalWebotChatAgent(ConversationalChatAgent):  # fixme: change it to Agent will leads to bug, but why?
    """An agent designed to hold a conversation in addition to using plugin tool."""

    output_parser: ConversationOutputParser = Field(
        default_factory=ConversationOutputParser()
    )

    template_tool_response: str = TEMPLATE_TOOL_RESPONSE
    continue_model: Optional[str] = None

    class Config:
        """Configuration for this pydantic object."""

        extra = Extra.allow
        arbitrary_types_allowed = True

    @classmethod
    def _get_default_output_parser(
            cls, **kwargs: Any
    ) -> ConversationOutputParser:
        return ConversationOutputParser()

    @property
    def _agent_type(self) -> str:
        raise NotImplementedError

    @property
    def observation_prefix(self) -> str:
        """Prefix to append the observation with."""
        return "Observation: "

    @property
    def llm_prefix(self) -> str:
        """Prefix to append the llm call with."""
        return "Thought:"


from real_agents.data_agent.copilot import ConversationalChatAgent

class ConversationalWebotChatAgent(ConversationalChatAgent):  # fixme: change it to Agent will leads to bug, but why?
    """An agent designed to hold a conversation in addition to using plugin tool."""

    output_parser: ConversationOutputParser = Field(
        default_factory=ConversationOutputParser()
    )

    template_tool_response: str = TEMPLATE_TOOL_RESPONSE
    continue_model: Optional[str] = None

    class Config:
        """Configuration for this pydantic object."""

        extra = Extra.allow
        arbitrary_types_allowed = True

    @classmethod
    def _get_default_output_parser(
            cls, **kwargs: Any
    ) -> ConversationOutputParser:
        return ConversationOutputParser()

    @property
    def _agent_type(self) -> str:
        raise NotImplementedError

    @property
    def observation_prefix(self) -> str:
        """Prefix to append the observation with."""
        return "Observation: "

    @property
    def llm_prefix(self) -> str:
        """Prefix to append the llm call with."""
        return "Thought:"

    @classmethod
    def _validate_tools(cls, tools: Sequence[BaseTool]) -> None:
        super()._validate_tools(tools)
        validate_tools_single_input(cls.__name__, tools)


    output_parser: ConversationOutputParser = Field(
        default_factory=ConversationOutputParser()
    )

    template_tool_response: str = TEMPLATE_TOOL_RESPONSE
    continue_model: Optional[str] = None

    class Config:
        """Configuration for this pydantic object."""

        extra = Extra.allow
        arbitrary_types_allowed = True

    @classmethod
    def _get_default_output_parser(
            cls, **kwargs: Any
    ) -> ConversationOutputParser:
        return ConversationOutputParser()

    @property
    def _agent_type(self) -> str:
        raise NotImplementedError

    @property
    def observation_prefix(self) -> str:
        """Prefix to append the observation with."""
        return "Observation: "

    @property
    def llm_prefix(self) -> str:
        """Prefix to append the llm call with."""
        return "Thought:"

    @classmethod
    def _validate_tools(cls, tools: Sequence[BaseTool]) -> None:
        super()._validate_tools(tools)
        validate_tools_single_input(cls.__name__, tools)

    @classmethod
    def create_prompt(
            cls,
            tools: Sequence[BaseTool],


    template_tool_response: str = TEMPLATE_TOOL_RESPONSE
    continue_model: Optional[str] = None

    class Config:
        """Configuration for this pydantic object."""

        extra = Extra.allow
        arbitrary_types_allowed = True

    @classmethod
    def _get_default_output_parser(
            cls, **kwargs: Any
    ) -> ConversationOutputParser:
        return ConversationOutputParser()

    @property
    def _agent_type(self) -> str:
        raise NotImplementedError

    @property
    def observation_prefix(self) -> str:
        """Prefix to append the observation with."""
        return "Observation: "

    @property
    def llm_prefix(self) -> str:
        """Prefix to append the llm call with."""
        return "Thought:"

    @classmethod
    def _validate_tools(cls, tools: Sequence[BaseTool]) -> None:
        super()._validate_tools(tools)
        validate_tools_single_input(cls.__name__, tools)

    @classmethod
    def create_prompt(
            cls,
            tools: Sequence[BaseTool],
            system_message: str = PREFIX,
            human_message: str = SUFFIX,
            input_variables: Optional[List[str]] = None,
            output_parser: Optional[BaseOutputParser] = None,
    ) -> BasePromptTemplate:


        extra = Extra.allow
        arbitrary_types_allowed = True

    @classmethod
    def _get_default_output_parser(
            cls, **kwargs: Any
    ) -> ConversationOutputParser:
        return ConversationOutputParser()

    @property
    def _agent_type(self) -> str:
        raise NotImplementedError

    @property
    def observation_prefix(self) -> str:
        """Prefix to append the observation with."""
        return "Observation: "

    @property
    def llm_prefix(self) -> str:
        """Prefix to append the llm call with."""
        return "Thought:"

    @classmethod
    def _validate_tools(cls, tools: Sequence[BaseTool]) -> None:
        super()._validate_tools(tools)
        validate_tools_single_input(cls.__name__, tools)

    @classmethod
    def create_prompt(
            cls,
            tools: Sequence[BaseTool],
            system_message: str = PREFIX,
            human_message: str = SUFFIX,
            input_variables: Optional[List[str]] = None,
            output_parser: Optional[BaseOutputParser] = None,
    ) -> BasePromptTemplate:
        tool_strings = "\n".join([f"Name: {tool.name}\nDescription: {tool.description}" for tool in tools])

        tool_names = ", ".join([tool.name for tool in tools])
        _output_parser = output_parser or cls._get_default_output_parser()


    def _get_default_output_parser(
            cls, **kwargs: Any
    ) -> ConversationOutputParser:
        return ConversationOutputParser()

    @property
    def _agent_type(self) -> str:
        raise NotImplementedError

    @property
    def observation_prefix(self) -> str:
        """Prefix to append the observation with."""
        return "Observation: "

    @property
    def llm_prefix(self) -> str:
        """Prefix to append the llm call with."""
        return "Thought:"

    @classmethod
    def _validate_tools(cls, tools: Sequence[BaseTool]) -> None:
        super()._validate_tools(tools)
        validate_tools_single_input(cls.__name__, tools)

    @classmethod
    def create_prompt(
            cls,
            tools: Sequence[BaseTool],
            system_message: str = PREFIX,
            human_message: str = SUFFIX,
            input_variables: Optional[List[str]] = None,
            output_parser: Optional[BaseOutputParser] = None,
    ) -> BasePromptTemplate:
        tool_strings = "\n".join([f"Name: {tool.name}\nDescription: {tool.description}" for tool in tools])

        tool_names = ", ".join([tool.name for tool in tools])
        _output_parser = output_parser or cls._get_default_output_parser()

        format_instructions = _output_parser.get_format_instructions("webot")
        format_instructions = format_instructions.format(tool_names=tool_names)

        # system message
        system_message = system_message + f"{tool_strings}\n\n{format_instructions}"


    @property
    def _agent_type(self) -> str:
        raise NotImplementedError

    @property
    def observation_prefix(self) -> str:
        """Prefix to append the observation with."""
        return "Observation: "

    @property
    def llm_prefix(self) -> str:
        """Prefix to append the llm call with."""
        return "Thought:"

    @classmethod
    def _validate_tools(cls, tools: Sequence[BaseTool]) -> None:
        super()._validate_tools(tools)
        validate_tools_single_input(cls.__name__, tools)

    @classmethod
    def create_prompt(
            cls,
            tools: Sequence[BaseTool],
            system_message: str = PREFIX,
            human_message: str = SUFFIX,
            input_variables: Optional[List[str]] = None,
            output_parser: Optional[BaseOutputParser] = None,
    ) -> BasePromptTemplate:
        tool_strings = "\n".join([f"Name: {tool.name}\nDescription: {tool.description}" for tool in tools])

        tool_names = ", ".join([tool.name for tool in tools])
        _output_parser = output_parser or cls._get_default_output_parser()

        format_instructions = _output_parser.get_format_instructions("webot")
        format_instructions = format_instructions.format(tool_names=tool_names)

        # system message
        system_message = system_message + f"{tool_strings}\n\n{format_instructions}"

        # human input
        final_prompt = human_message

        if input_variables is None:
            input_variables = ["input", "chat_history", "agent_scratchpad"]
        messages = [
            SystemMessagePromptTemplate.from_template(system_message),
            MessagesPlaceholder(variable_name="chat_history"),
            HumanMessagePromptTemplate.from_template(final_prompt),
            MessagesPlaceholder(variable_name="agent_scratchpad"),
        ]
        return ChatPromptTemplate(input_variables=input_variables, messages=messages)

    def _construct_scratchpad(self, intermediate_steps: List[Tuple[AgentAction, str]]) -> List[BaseMessage]:
        """Construct the scratchpad that lets the agent continue its thought process."""
        thoughts: List[BaseMessage] = []
        # Try to only use AI message for scratchpad
        content = []
        for idx, (action, full_observation) in enumerate(intermediate_steps):
            content.append(MessageDataModel.extract_action_for_llm(action.log))

            observation = full_observation
            if isinstance(full_observation, DataModel):
                llm_raw_observation = full_observation.get_llm_side_data()

                observation = MessageDataModel.extract_tool_response_for_llm(llm_raw_observation, tool_style="plugin")

            if idx == len(intermediate_steps) - 1:
                tool_response = self.template_tool_response.format(
                    observation=str(observation), tool_names=self.allowed_tools
                )


        format_instructions = _output_parser.get_format_instructions("webot")
        format_instructions = format_instructions.format(tool_names=tool_names)

        # system message
        system_message = system_message + f"{tool_strings}\n\n{format_instructions}"

        # human input
        final_prompt = human_message

        if input_variables is None:
            input_variables = ["input", "chat_history", "agent_scratchpad"]
        messages = [
            SystemMessagePromptTemplate.from_template(system_message),
            MessagesPlaceholder(variable_name="chat_history"),
            HumanMessagePromptTemplate.from_template(final_prompt),
            MessagesPlaceholder(variable_name="agent_scratchpad"),
        ]
        return ChatPromptTemplate(input_variables=input_variables, messages=messages)

    def _construct_scratchpad(self, intermediate_steps: List[Tuple[AgentAction, str]]) -> List[BaseMessage]:
        """Construct the scratchpad that lets the agent continue its thought process."""
        thoughts: List[BaseMessage] = []
        # Try to only use AI message for scratchpad
        content = []
        for idx, (action, full_observation) in enumerate(intermediate_steps):
            content.append(MessageDataModel.extract_action_for_llm(action.log))

            observation = full_observation
            if isinstance(full_observation, DataModel):
                llm_raw_observation = full_observation.get_llm_side_data()

                observation = MessageDataModel.extract_tool_response_for_llm(llm_raw_observation, tool_style="plugin")

            if idx == len(intermediate_steps) - 1:
                tool_response = self.template_tool_response.format(
                    observation=str(observation), tool_names=self.allowed_tools
                )
                if idx == len(intermediate_steps) - 1:
                    content.append(tool_response)
                else:
                    content.append(observation)

        content_str = "\n".join(content)
        thoughts.append(AIMessage(content=content_str))
        if self.continue_model is not None and len(intermediate_steps) != 0:
            thoughts.append(HumanMessage(content=fake_continue_prompt[self.continue_model]))
        return thoughts

    @override
    def plan(
            self,
            intermediate_steps: List[Tuple[AgentAction, str]],
            **kwargs: Any,
    ) -> Union[AgentAction, AgentFinish]:
        """Given input, decided what to do.

        Args:
            intermediate_steps: Steps the LLM has taken to date,
                along with observations
            **kwargs: User inputs.

        Returns:
            Action specifying what tool to use.
        """
        full_inputs = self.get_full_inputs(intermediate_steps, **kwargs)
        system_prompt = self.llm_chain.prompt.messages[0].format().content
        system_prompt_tokens = MessageDataModel._count_tokens(


plan

                llm_raw_observation = full_observation.get_llm_side_data()

                observation = MessageDataModel.extract_tool_response_for_llm(llm_raw_observation, tool_style="plugin")

            if idx == len(intermediate_steps) - 1:
                tool_response = self.template_tool_response.format(
                    observation=str(observation), tool_names=self.allowed_tools
                )
                if idx == len(intermediate_steps) - 1:
                    content.append(tool_response)
                else:
                    content.append(observation)

        content_str = "\n".join(content)
        thoughts.append(AIMessage(content=content_str))
        if self.continue_model is not None and len(intermediate_steps) != 0:
            thoughts.append(HumanMessage(content=fake_continue_prompt[self.continue_model]))
        return thoughts

    @override
    def plan(
            self,
            intermediate_steps: List[Tuple[AgentAction, str]],
            **kwargs: Any,
    ) -> Union[AgentAction, AgentFinish]:
        """Given input, decided what to do.

        Args:
            intermediate_steps: Steps the LLM has taken to date,
                along with observations
            **kwargs: User inputs.

        Returns:
            Action specifying what tool to use.
        """
        full_inputs = self.get_full_inputs(intermediate_steps, **kwargs)
        system_prompt = self.llm_chain.prompt.messages[0].format().content
        system_prompt_tokens = MessageDataModel._count_tokens(
            system_prompt
        )
        max_tokens = 8000
        max_gen_tokens = 1000
        # FIXME: need more accurate token limit calculation
        full_inputs = MessageDataModel.truncate_chat_history(full_inputs, max_token=max_tokens - system_prompt_tokens - max_gen_tokens)
        full_output = self.llm_chain.predict(**full_inputs)

        return self.output_parser.parse(full_output)

    @classmethod
    def from_llm_and_tools(
            cls,
            llm: BaseLanguageModel,
            tools: Sequence[BaseTool],
            callbacks: Callbacks = None,
            output_parser: Optional[AgentOutputParser] = None,
            system_message: str = PREFIX,
            human_message: str = SUFFIX,
            input_variables: Optional[List[str]] = None,
            **kwargs: Any,
    ) -> Agent:
        """Construct an agent from an LLM and tools."""
        cls._validate_tools(tools)

        _output_parser = output_parser or cls._get_default_output_parser()
        prompt = cls.create_prompt(
            tools,
            system_message=system_message,


                along with observations
            **kwargs: User inputs.

        Returns:
            Action specifying what tool to use.
        """
        full_inputs = self.get_full_inputs(intermediate_steps, **kwargs)
        system_prompt = self.llm_chain.prompt.messages[0].format().content
        system_prompt_tokens = MessageDataModel._count_tokens(
            system_prompt
        )
        max_tokens = 8000
        max_gen_tokens = 1000
        # FIXME: need more accurate token limit calculation
        full_inputs = MessageDataModel.truncate_chat_history(full_inputs, max_token=max_tokens - system_prompt_tokens - max_gen_tokens)
        full_output = self.llm_chain.predict(**full_inputs)

        return self.output_parser.parse(full_output)

    @classmethod
    def from_llm_and_tools(
            cls,
            llm: BaseLanguageModel,
            tools: Sequence[BaseTool],
            callbacks: Callbacks = None,
            output_parser: Optional[AgentOutputParser] = None,
            system_message: str = PREFIX,
            human_message: str = SUFFIX,
            input_variables: Optional[List[str]] = None,
            **kwargs: Any,
    ) -> Agent:
        """Construct an agent from an LLM and tools."""
        cls._validate_tools(tools)

        _output_parser = output_parser or cls._get_default_output_parser()
        prompt = cls.create_prompt(
            tools,
            system_message=system_message,
            human_message=human_message,
            input_variables=input_variables,
            output_parser=_output_parser,
        )
        llm_chain = LLMChain(
            llm=llm,
            prompt=prompt,
        )
        tool_names = [tool.name for tool in tools]
        return cls(
            llm_chain=llm_chain,
            allowed_tools=tool_names,
            output_parser=_output_parser,
            **kwargs,
        )


"""
Implementation of the WebBrowsingExecutor.
WebBrowsingExecutor takes start_url and instruction as input, iteratively perform the actions on the web, and return the result.
"""
from typing import Any, Dict, List

from langchain.base_language import BaseLanguageModel

from real_agents.web_agent.web_browsing.react.base import ReActWebotChain
from real_agents.web_agent.web_browsing.end2end.base import WebotChain
from real_agents.adapters.data_model.html import HTMLDataModel

# This executor is for the extension usage and not for the chat interface.
# For the chat interface webot executor, refer xlang/real_agents/web_agent/executors/web_browsing_executor.py
class WebBrowsingExecutor:
    """
    WebBrowsingExecutor takes start_url and instruction as input, iteratively perform the actions on the web, and return the result.
    """

    def __init__(self, instruction: str, plan: str = "", mode: str = "react") -> None:
        """Initialize the executor"""
        self.instruction: str = instruction
        self.mode: str = mode
        if self.mode == "react":
            self.thoughts_taken: List[str] = []
        self.actions_taken: List[str] = []
        self.pages_viewed: List[Any] = []
        self.plan: str = plan

    @property
    def finish(self):
        return True if len(self.actions_taken) > 0 and "finish" in self.actions_taken[-1] else False

    @property
    def interrupt(self):
        return True if len(self.actions_taken) > 0 and "interrupt" in self.actions_taken[-1] else False

    @property
    def error(self):
        return True if len(self.actions_taken) > 0 and "error" in self.actions_taken[-1] else False

    @property
    def fail(self):
        return True if len(self.actions_taken) > 0 and "fail" in self.actions_taken[-1] else False

    @property
    def action_history(self):
        if self.mode == "basic":
            action_history = "Action: "
            for action in self.actions_taken:
                action_history += action + " -> "
            return action_history
        elif self.mode == "react":
            action_history = ""
            for thought, action in zip(self.thoughts_taken, self.actions_taken):
                action_history += thought + " -> " + action + " -> "
            return action_history
        else:
            raise ValueError(f"The mode {self.mode} is not supported")

    def run(
            self,
            page_info: Any,
            llm: BaseLanguageModel,
    ) -> Dict[str, Any]:
        model = HTMLDataModel.from_raw_data(raw_data=page_info)
        processed_html = model.get_llm_side_data()
        if self.mode == "basic":
            method = WebotChain.from_llm(llm)
            self.pages_viewed.append(processed_html)
            action_element = method(
                {"user_query": self.instruction, "previous_actions": self.actions_taken, "page_info": processed_html}
            )
        elif self.mode == "react":
            method = ReActWebotChain.from_llm(llm)
            self.pages_viewed.append(processed_html)
            print("self.plan:", self.plan)

            # example: {'success': True, 'message' = 'success', 'thought': "I should first set the value in the search field to '...'", 'action': 'setValue(93, "...")', 'parsedAction': {'name': 'setValue', 'args': {'elementId': 93, 'value': '...'}}}
            webot_chain_return = method(
                {
                    "user_query": self.instruction,
                    "plan": self.plan,
                    "previous_actions": self.actions_taken,
                    "previous_thoughts": self.thoughts_taken,
                    "page_info": processed_html,
                }
            )
        else:
            raise ValueError(f"The mode {self.mode} is not supported")

        # "I should first set the value in the search field to '...'"
        self.thoughts_taken.append(webot_chain_return["thought"])

        # setValue(93, "...")
        self.actions_taken.append(webot_chain_return["action"])

        print("actions_taken:", self.actions_taken)
        return webot_chain_return


"""
Implementation of the WebBrowsingExecutor.
WebBrowsingExecutor takes start_url and instruction as input, iteratively perform the actions on the web, and return the result.
"""
from typing import Any, Dict, List

from langchain.base_language import BaseLanguageModel

from real_agents.web_agent.web_browsing.react.base import ReActWebotChain
from real_agents.web_agent.web_browsing.end2end.base import WebotChain
from real_agents.adapters.data_model.html import HTMLDataModel

# This executor is for the extension usage and not for the chat interface.
# For the chat interface webot executor, refer xlang/real_agents/web_agent/executors/web_browsing_executor.py
class WebBrowsingExecutor:
    """
    WebBrowsingExecutor takes start_url and instruction as input, iteratively perform the actions on the web, and return the result.
    """

    def __init__(self, instruction: str, plan: str = "", mode: str = "react") -> None:
        """Initialize the executor"""
        self.instruction: str = instruction
        self.mode: str = mode
        if self.mode == "react":
            self.thoughts_taken: List[str] = []
        self.actions_taken: List[str] = []
        self.pages_viewed: List[Any] = []
        self.plan: str = plan

    @property
    def finish(self):
        return True if len(self.actions_taken) > 0 and "finish" in self.actions_taken[-1] else False

    @property
    def interrupt(self):
        return True if len(self.actions_taken) > 0 and "interrupt" in self.actions_taken[-1] else False

    @property
    def error(self):
        return True if len(self.actions_taken) > 0 and "error" in self.actions_taken[-1] else False

    @property
    def fail(self):
        return True if len(self.actions_taken) > 0 and "fail" in self.actions_taken[-1] else False

    @property
    def action_history(self):
        if self.mode == "basic":


finish

# This executor is for the extension usage and not for the chat interface.
# For the chat interface webot executor, refer xlang/real_agents/web_agent/executors/web_browsing_executor.py
class WebBrowsingExecutor:
    """
    WebBrowsingExecutor takes start_url and instruction as input, iteratively perform the actions on the web, and return the result.
    """

    def __init__(self, instruction: str, plan: str = "", mode: str = "react") -> None:
        """Initialize the executor"""
        self.instruction: str = instruction
        self.mode: str = mode
        if self.mode == "react":
            self.thoughts_taken: List[str] = []
        self.actions_taken: List[str] = []
        self.pages_viewed: List[Any] = []
        self.plan: str = plan

    @property
    def finish(self):
        return True if len(self.actions_taken) > 0 and "finish" in self.actions_taken[-1] else False

    @property
    def interrupt(self):
        return True if len(self.actions_taken) > 0 and "interrupt" in self.actions_taken[-1] else False

    @property
    def error(self):
        return True if len(self.actions_taken) > 0 and "error" in self.actions_taken[-1] else False

    @property
    def fail(self):
        return True if len(self.actions_taken) > 0 and "fail" in self.actions_taken[-1] else False

    @property
    def action_history(self):
        if self.mode == "basic":
            action_history = "Action: "
            for action in self.actions_taken:
                action_history += action + " -> "
            return action_history


interrupt

class WebBrowsingExecutor:
    """
    WebBrowsingExecutor takes start_url and instruction as input, iteratively perform the actions on the web, and return the result.
    """

    def __init__(self, instruction: str, plan: str = "", mode: str = "react") -> None:
        """Initialize the executor"""
        self.instruction: str = instruction
        self.mode: str = mode
        if self.mode == "react":
            self.thoughts_taken: List[str] = []
        self.actions_taken: List[str] = []
        self.pages_viewed: List[Any] = []
        self.plan: str = plan

    @property
    def finish(self):
        return True if len(self.actions_taken) > 0 and "finish" in self.actions_taken[-1] else False

    @property
    def interrupt(self):
        return True if len(self.actions_taken) > 0 and "interrupt" in self.actions_taken[-1] else False

    @property
    def error(self):
        return True if len(self.actions_taken) > 0 and "error" in self.actions_taken[-1] else False

    @property
    def fail(self):
        return True if len(self.actions_taken) > 0 and "fail" in self.actions_taken[-1] else False

    @property
    def action_history(self):
        if self.mode == "basic":
            action_history = "Action: "
            for action in self.actions_taken:
                action_history += action + " -> "
            return action_history
        elif self.mode == "react":
            action_history = ""
            for thought, action in zip(self.thoughts_taken, self.actions_taken):
                action_history += thought + " -> " + action + " -> "


error

    def __init__(self, instruction: str, plan: str = "", mode: str = "react") -> None:
        """Initialize the executor"""
        self.instruction: str = instruction
        self.mode: str = mode
        if self.mode == "react":
            self.thoughts_taken: List[str] = []
        self.actions_taken: List[str] = []
        self.pages_viewed: List[Any] = []
        self.plan: str = plan

    @property
    def finish(self):
        return True if len(self.actions_taken) > 0 and "finish" in self.actions_taken[-1] else False

    @property
    def interrupt(self):
        return True if len(self.actions_taken) > 0 and "interrupt" in self.actions_taken[-1] else False

    @property
    def error(self):
        return True if len(self.actions_taken) > 0 and "error" in self.actions_taken[-1] else False

    @property
    def fail(self):
        return True if len(self.actions_taken) > 0 and "fail" in self.actions_taken[-1] else False

    @property
    def action_history(self):
        if self.mode == "basic":
            action_history = "Action: "
            for action in self.actions_taken:
                action_history += action + " -> "
            return action_history
        elif self.mode == "react":
            action_history = ""
            for thought, action in zip(self.thoughts_taken, self.actions_taken):
                action_history += thought + " -> " + action + " -> "
            return action_history
        else:
            raise ValueError(f"The mode {self.mode} is not supported")


fail

        self.mode: str = mode
        if self.mode == "react":
            self.thoughts_taken: List[str] = []
        self.actions_taken: List[str] = []
        self.pages_viewed: List[Any] = []
        self.plan: str = plan

    @property
    def finish(self):
        return True if len(self.actions_taken) > 0 and "finish" in self.actions_taken[-1] else False

    @property
    def interrupt(self):
        return True if len(self.actions_taken) > 0 and "interrupt" in self.actions_taken[-1] else False

    @property
    def error(self):
        return True if len(self.actions_taken) > 0 and "error" in self.actions_taken[-1] else False

    @property
    def fail(self):
        return True if len(self.actions_taken) > 0 and "fail" in self.actions_taken[-1] else False

    @property
    def action_history(self):
        if self.mode == "basic":
            action_history = "Action: "
            for action in self.actions_taken:
                action_history += action + " -> "
            return action_history
        elif self.mode == "react":
            action_history = ""
            for thought, action in zip(self.thoughts_taken, self.actions_taken):
                action_history += thought + " -> " + action + " -> "
            return action_history
        else:
            raise ValueError(f"The mode {self.mode} is not supported")

    def run(
            self,
            page_info: Any,
            llm: BaseLanguageModel,