import re
import textwrap
from typing import List, Dict, Any, Optional
from langchain.schema import BaseMessage
import tiktoken

# format of agent action
ACTION_FORMAT = """```json
{{
    "action": "{_action}",
    "action_input": "{_action_input}",
}}
```"""

# format of tool call(code) & tool output(response)
TOOL_FORMAT = {
    "code": """<code>
{_intermediate_steps}
</code>

<output>
{_result}
</output>
""",
    "plugin": """<plugin_call>
{_intermediate_steps}
</plugin_call>

<output>
{_result}
</output>
""",
}

# format to wrap tool call + tool output together
TOOL_RESPONSE_FORMAT = """[RESPONSE_BEGIN]
{_response}
[RESPONSE_END]
"""

class MessageDataModel:
    """A data model for Message Management, general purpose."""

    @staticmethod
    def _count_tokens(test_string: str) -> int:
        """copy of langchain _get_num_token_default_method"""
        enc = tiktoken.get_encoding("cl100k_base")
        tokens = len(enc.encode(test_string))
        return tokens

    @classmethod
    def _get_num_tokens_from_messages(cls, buffer: List[BaseMessage]) -> int:
        return sum([cls._count_tokens(m.content) for m in buffer])

    @classmethod
    def truncate_text(cls, raw_text: str, max_token: Optional[int] = 250, trunc_ratio: int = 0.5) -> str:
        """heuristic truncation for single long string & code"""
        tokens = cls._count_tokens(raw_text)
        if max_token is None or tokens <= max_token:
            return raw_text

        # assume we keep the first ratio * max_tokens and the (1 - ratio) * max_tokens
        half_tokens = int(max_token * trunc_ratio)
        lines = raw_text.strip().split("\n")
        lines = [" ".join(line.split(" ")[:100]) for line in lines]
        total_lines = len(lines)

        # first half
        left = 0
        right = total_lines // 2
        while left < right:
            mid = (left + right) >> 1
            text = "\n".join(lines[0:mid])
            token = cls._count_tokens(text)
            if token > half_tokens:
                right = mid
            else:
                left = mid + 1
        first_half = "\n".join(lines[0:right])

        # last half
        left = total_lines // 2 + 1
        right = total_lines - 1
        while left < right:
            mid = (left + right) >> 1
            text = "\n".join(lines[mid:])
            token = cls._count_tokens(text)
            if token > half_tokens:
                right = mid
            else:
                left = mid + 1
        second_half = "\n".join(lines[left:])

        if first_half != "" or second_half != "":
            return f"{first_half}\n...\n[too long to show]\n...\n{second_half}"
        else:
            # if len(first_half_list) == 0 and len(last_half_list) == 0:
            # if all lines >= max_token, return last 100 words as truncated results.
            return f"...\n[too long to show]\n...\n{raw_text[-100:]}"

    @classmethod
    def truncate_chat_history(cls, full_inputs: Dict[str, Any], max_token: int = 2500) -> Dict[str, Any]:
        _input = full_inputs["input"]
        agent_scratchpad = full_inputs["agent_scratchpad"]
        agent_scratchpad = "\n".join([_.content for _ in agent_scratchpad])
        _input_tokens = cls._count_tokens(_input)
        _scratchpad_tokens = cls._count_tokens(agent_scratchpad)

        left_tokens = max_token - _scratchpad_tokens - _input_tokens
        chat_history = full_inputs["chat_history"]

        curr_buffer_length = cls._get_num_tokens_from_messages(chat_history)
        while len(chat_history) != 0 and curr_buffer_length > left_tokens:
            chat_history.pop(0)
            curr_buffer_length = cls._get_num_tokens_from_messages(chat_history)
        full_inputs["chat_history"] = chat_history
        return full_inputs

    @staticmethod
    def _extract_value(json_string: str, key: str) -> str:
        pattern = re.compile(rf'"?{key}"?\s*:\s*("((?:[^"\\]|\\.)*)"|(\b[^,\s]*\b))', re.MULTILINE)
        match = pattern.search(json_string)
        if match:
            result = match.group(1).replace('\\"', '"').replace("\\\\", "\\").strip('"').strip("'").strip()
            # result = f"\"{result}\""
            return result
        raise ValueError(f"Could not find {key} in {json_string}")

    @staticmethod
    def _extract_response(
        chat_history: str,
        begin_marker: str = "[RESPONSE_BEGIN]",
        end_marker: str = "[RESPONSE_END]",
        ai_msg_marker: str = "AI:",
    ):
        code_blocks = chat_history.split(ai_msg_marker)
        pattern = r"\[RESPONSE_BEGIN\](.*?)\[RESPONSE_END\]"

        cleaned_output = []
        for code_block in code_blocks:
            matches = re.findall(pattern, code_block, re.DOTALL)
            if matches:
                cleaned_output.append(matches[0].strip())
        return "\n".join(cleaned_output)

    @classmethod
    def extract_action_for_llm(cls, text, max_token: int = 500) -> str:
        """Since Action should be fully inputted into an Agent, so we do not perform truncation here."""
        action_format = ACTION_FORMAT
        cleaned_output = text.strip()
        try:
            _action = cls._extract_value(cleaned_output, "action")
            _action_input = cls._extract_value(cleaned_output, "action_input")
            return action_format.format(_action=_action, _action_input=_action_input)
        except Exception:
            if cleaned_output.startswith("Action:"):
                lines = cleaned_output.splitlines()
                _action = lines[1].strip()
                _action_input = textwrap.dedent("\n".join(lines[2:])).strip()
                return action_format.format(_action=_action, _action_input=_action_input)
            else:
                _action_input = cleaned_output

            return action_format.format(_action="Final Answer", _action_input=_action_input)

    @classmethod
    def extract_tool_response_for_llm(cls, text, tool_style: str = "code", max_token: int = 250) -> str:
        wrap_format = TOOL_RESPONSE_FORMAT
        tool_observation_format = TOOL_FORMAT[tool_style]
        cleaned_output = text.strip()
        if tool_style == "plugin":
            max_token = None

        try:
            _result = cls.truncate_text(cls._extract_value(cleaned_output, "result"), max_token)
            _intermediate_steps = cls.truncate_text(
                cls._extract_value(cleaned_output, "intermediate_steps"), max_token
            )
            _intermediate_steps = _intermediate_steps.replace("\\n", "\n").strip("\n")
            _result = _result.replace("\\n", "\n").strip("\n")
            _response = tool_observation_format.format(_intermediate_steps=_intermediate_steps, _result=_result)

            return wrap_format.format(_response=_response)
        except:
            if cleaned_output.startswith("Final Answer:"):
                lines = cleaned_output.splitlines()
                _response = textwrap.dedent("\n".join(lines[2:])).strip()
                _response = cls.truncate_text(_response, max_token)
                return wrap_format.format(_response=_response)

            _response = cls.truncate_text(cleaned_output, max_token)
            return wrap_format.format(_response=_response)

    @classmethod
    def extract_code_for_python_tool(cls, text: str, max_token: int = 2500, trunc_ratio: float = 0.2) -> str:
        whole_code = MessageDataModel._extract_response(text)
        trunc_code = cls.truncate_text(whole_code, max_token=max_token, trunc_ratio=trunc_ratio)
        return trunc_code

    @classmethod
    def extract_code_for_sql_tool(cls, text: str, max_token: int = 2500, trunc_ratio: float = 0.2) -> str:
        whole_code = MessageDataModel._extract_response(text)
        trunc_code = cls.truncate_text(whole_code, max_token=max_token, trunc_ratio=trunc_ratio)
        return trunc_code


MessageDataModel

{_result}
</output>
""",
    "plugin": """<plugin_call>
{_intermediate_steps}
</plugin_call>

<output>
{_result}
</output>
""",
}

# format to wrap tool call + tool output together
TOOL_RESPONSE_FORMAT = """[RESPONSE_BEGIN]
{_response}
[RESPONSE_END]
"""

class MessageDataModel:
    """A data model for Message Management, general purpose."""

    @staticmethod
    def _count_tokens(test_string: str) -> int:
        """copy of langchain _get_num_token_default_method"""
        enc = tiktoken.get_encoding("cl100k_base")
        tokens = len(enc.encode(test_string))
        return tokens

    @classmethod
    def _get_num_tokens_from_messages(cls, buffer: List[BaseMessage]) -> int:
        return sum([cls._count_tokens(m.content) for m in buffer])

    @classmethod
    def truncate_text(cls, raw_text: str, max_token: Optional[int] = 250, trunc_ratio: int = 0.5) -> str:
        """heuristic truncation for single long string & code"""
        tokens = cls._count_tokens(raw_text)
        if max_token is None or tokens <= max_token:
            return raw_text

        # assume we keep the first ratio * max_tokens and the (1 - ratio) * max_tokens
        half_tokens = int(max_token * trunc_ratio)
        lines = raw_text.strip().split("\n")
        lines = [" ".join(line.split(" ")[:100]) for line in lines]
        total_lines = len(lines)

        # first half
        left = 0
        right = total_lines // 2
        while left < right:
            mid = (left + right) >> 1
            text = "\n".join(lines[0:mid])
            token = cls._count_tokens(text)
            if token > half_tokens:
                right = mid
            else:
                left = mid + 1
        first_half = "\n".join(lines[0:right])

        # last half
        left = total_lines // 2 + 1
        right = total_lines - 1
        while left < right:
            mid = (left + right) >> 1
            text = "\n".join(lines[mid:])
            token = cls._count_tokens(text)
            if token > half_tokens:
                right = mid
            else:
                left = mid + 1
        second_half = "\n".join(lines[left:])

        if first_half != "" or second_half != "":
            return f"{first_half}\n...\n[too long to show]\n...\n{second_half}"
        else:
            # if len(first_half_list) == 0 and len(last_half_list) == 0:
            # if all lines >= max_token, return last 100 words as truncated results.
            return f"...\n[too long to show]\n...\n{raw_text[-100:]}"

    @classmethod
    def truncate_chat_history(cls, full_inputs: Dict[str, Any], max_token: int = 2500) -> Dict[str, Any]:
        _input = full_inputs["input"]
        agent_scratchpad = full_inputs["agent_scratchpad"]
        agent_scratchpad = "\n".join([_.content for _ in agent_scratchpad])
        _input_tokens = cls._count_tokens(_input)
        _scratchpad_tokens = cls._count_tokens(agent_scratchpad)

        left_tokens = max_token - _scratchpad_tokens - _input_tokens
        chat_history = full_inputs["chat_history"]

        curr_buffer_length = cls._get_num_tokens_from_messages(chat_history)
        while len(chat_history) != 0 and curr_buffer_length > left_tokens:
            chat_history.pop(0)
            curr_buffer_length = cls._get_num_tokens_from_messages(chat_history)
        full_inputs["chat_history"] = chat_history
        return full_inputs

    @staticmethod
    def _extract_value(json_string: str, key: str) -> str:
        pattern = re.compile(rf'"?{key}"?\s*:\s*("((?:[^"\\]|\\.)*)"|(\b[^,\s]*\b))', re.MULTILINE)
        match = pattern.search(json_string)
        if match:
            result = match.group(1).replace('\\"', '"').replace("\\\\", "\\").strip('"').strip("'").strip()
            # result = f"\"{result}\""
            return result
        raise ValueError(f"Could not find {key} in {json_string}")

    @staticmethod
    def _extract_response(
        chat_history: str,
        begin_marker: str = "[RESPONSE_BEGIN]",
        end_marker: str = "[RESPONSE_END]",
        ai_msg_marker: str = "AI:",
    ):
        code_blocks = chat_history.split(ai_msg_marker)
        pattern = r"\[RESPONSE_BEGIN\](.*?)\[RESPONSE_END\]"

        cleaned_output = []
        for code_block in code_blocks:
            matches = re.findall(pattern, code_block, re.DOTALL)
            if matches:
                cleaned_output.append(matches[0].strip())
        return "\n".join(cleaned_output)

    @classmethod
    def extract_action_for_llm(cls, text, max_token: int = 500) -> str:
        """Since Action should be fully inputted into an Agent, so we do not perform truncation here."""
        action_format = ACTION_FORMAT
        cleaned_output = text.strip()
        try:
            _action = cls._extract_value(cleaned_output, "action")
            _action_input = cls._extract_value(cleaned_output, "action_input")
            return action_format.format(_action=_action, _action_input=_action_input)
        except Exception:
            if cleaned_output.startswith("Action:"):
                lines = cleaned_output.splitlines()
                _action = lines[1].strip()
                _action_input = textwrap.dedent("\n".join(lines[2:])).strip()
                return action_format.format(_action=_action, _action_input=_action_input)
            else:
                _action_input = cleaned_output

            return action_format.format(_action="Final Answer", _action_input=_action_input)

    @classmethod
    def extract_tool_response_for_llm(cls, text, tool_style: str = "code", max_token: int = 250) -> str:
        wrap_format = TOOL_RESPONSE_FORMAT
        tool_observation_format = TOOL_FORMAT[tool_style]
        cleaned_output = text.strip()
        if tool_style == "plugin":
            max_token = None

        try:
            _result = cls.truncate_text(cls._extract_value(cleaned_output, "result"), max_token)
            _intermediate_steps = cls.truncate_text(
                cls._extract_value(cleaned_output, "intermediate_steps"), max_token
            )
            _intermediate_steps = _intermediate_steps.replace("\\n", "\n").strip("\n")
            _result = _result.replace("\\n", "\n").strip("\n")
            _response = tool_observation_format.format(_intermediate_steps=_intermediate_steps, _result=_result)

            return wrap_format.format(_response=_response)
        except:
            if cleaned_output.startswith("Final Answer:"):
                lines = cleaned_output.splitlines()
                _response = textwrap.dedent("\n".join(lines[2:])).strip()
                _response = cls.truncate_text(_response, max_token)
                return wrap_format.format(_response=_response)

            _response = cls.truncate_text(cleaned_output, max_token)
            return wrap_format.format(_response=_response)

    @classmethod
    def extract_code_for_python_tool(cls, text: str, max_token: int = 2500, trunc_ratio: float = 0.2) -> str:
        whole_code = MessageDataModel._extract_response(text)
        trunc_code = cls.truncate_text(whole_code, max_token=max_token, trunc_ratio=trunc_ratio)
        return trunc_code

    @classmethod
    def extract_code_for_sql_tool(cls, text: str, max_token: int = 2500, trunc_ratio: float = 0.2) -> str:
        whole_code = MessageDataModel._extract_response(text)
        trunc_code = cls.truncate_text(whole_code, max_token=max_token, trunc_ratio=trunc_ratio)
        return trunc_code


{_intermediate_steps}
</plugin_call>

<output>
{_result}
</output>
""",
}

# format to wrap tool call + tool output together
TOOL_RESPONSE_FORMAT = """[RESPONSE_BEGIN]
{_response}
[RESPONSE_END]
"""

class MessageDataModel:
    """A data model for Message Management, general purpose."""

    @staticmethod
    def _count_tokens(test_string: str) -> int:
        """copy of langchain _get_num_token_default_method"""
        enc = tiktoken.get_encoding("cl100k_base")
        tokens = len(enc.encode(test_string))
        return tokens

    @classmethod
    def _get_num_tokens_from_messages(cls, buffer: List[BaseMessage]) -> int:
        return sum([cls._count_tokens(m.content) for m in buffer])

    @classmethod
    def truncate_text(cls, raw_text: str, max_token: Optional[int] = 250, trunc_ratio: int = 0.5) -> str:
        """heuristic truncation for single long string & code"""
        tokens = cls._count_tokens(raw_text)
        if max_token is None or tokens <= max_token:
            return raw_text

        # assume we keep the first ratio * max_tokens and the (1 - ratio) * max_tokens
        half_tokens = int(max_token * trunc_ratio)
        lines = raw_text.strip().split("\n")
        lines = [" ".join(line.split(" ")[:100]) for line in lines]
        total_lines = len(lines)

        # first half
        left = 0


}

# format to wrap tool call + tool output together
TOOL_RESPONSE_FORMAT = """[RESPONSE_BEGIN]
{_response}
[RESPONSE_END]
"""

class MessageDataModel:
    """A data model for Message Management, general purpose."""

    @staticmethod
    def _count_tokens(test_string: str) -> int:
        """copy of langchain _get_num_token_default_method"""
        enc = tiktoken.get_encoding("cl100k_base")
        tokens = len(enc.encode(test_string))
        return tokens

    @classmethod
    def _get_num_tokens_from_messages(cls, buffer: List[BaseMessage]) -> int:
        return sum([cls._count_tokens(m.content) for m in buffer])

    @classmethod
    def truncate_text(cls, raw_text: str, max_token: Optional[int] = 250, trunc_ratio: int = 0.5) -> str:
        """heuristic truncation for single long string & code"""
        tokens = cls._count_tokens(raw_text)
        if max_token is None or tokens <= max_token:
            return raw_text

        # assume we keep the first ratio * max_tokens and the (1 - ratio) * max_tokens
        half_tokens = int(max_token * trunc_ratio)
        lines = raw_text.strip().split("\n")
        lines = [" ".join(line.split(" ")[:100]) for line in lines]
        total_lines = len(lines)

        # first half
        left = 0
        right = total_lines // 2
        while left < right:
            mid = (left + right) >> 1
            text = "\n".join(lines[0:mid])


{_response}
[RESPONSE_END]
"""

class MessageDataModel:
    """A data model for Message Management, general purpose."""

    @staticmethod
    def _count_tokens(test_string: str) -> int:
        """copy of langchain _get_num_token_default_method"""
        enc = tiktoken.get_encoding("cl100k_base")
        tokens = len(enc.encode(test_string))
        return tokens

    @classmethod
    def _get_num_tokens_from_messages(cls, buffer: List[BaseMessage]) -> int:
        return sum([cls._count_tokens(m.content) for m in buffer])

    @classmethod
    def truncate_text(cls, raw_text: str, max_token: Optional[int] = 250, trunc_ratio: int = 0.5) -> str:
        """heuristic truncation for single long string & code"""
        tokens = cls._count_tokens(raw_text)
        if max_token is None or tokens <= max_token:
            return raw_text

        # assume we keep the first ratio * max_tokens and the (1 - ratio) * max_tokens
        half_tokens = int(max_token * trunc_ratio)
        lines = raw_text.strip().split("\n")
        lines = [" ".join(line.split(" ")[:100]) for line in lines]
        total_lines = len(lines)

        # first half
        left = 0
        right = total_lines // 2
        while left < right:
            mid = (left + right) >> 1
            text = "\n".join(lines[0:mid])
            token = cls._count_tokens(text)
            if token > half_tokens:
                right = mid
            else:
                left = mid + 1
        first_half = "\n".join(lines[0:right])

        # last half
        left = total_lines // 2 + 1
        right = total_lines - 1
        while left < right:
            mid = (left + right) >> 1
            text = "\n".join(lines[mid:])
            token = cls._count_tokens(text)
            if token > half_tokens:
                right = mid
            else:
                left = mid + 1
        second_half = "\n".join(lines[left:])

        if first_half != "" or second_half != "":
            return f"{first_half}\n...\n[too long to show]\n...\n{second_half}"
        else:
            # if len(first_half_list) == 0 and len(last_half_list) == 0:
            # if all lines >= max_token, return last 100 words as truncated results.
            return f"...\n[too long to show]\n...\n{raw_text[-100:]}"

    @classmethod
    def truncate_chat_history(cls, full_inputs: Dict[str, Any], max_token: int = 2500) -> Dict[str, Any]:
        _input = full_inputs["input"]
        agent_scratchpad = full_inputs["agent_scratchpad"]
        agent_scratchpad = "\n".join([_.content for _ in agent_scratchpad])
        _input_tokens = cls._count_tokens(_input)
        _scratchpad_tokens = cls._count_tokens(agent_scratchpad)

        left_tokens = max_token - _scratchpad_tokens - _input_tokens
        chat_history = full_inputs["chat_history"]

        curr_buffer_length = cls._get_num_tokens_from_messages(chat_history)
        while len(chat_history) != 0 and curr_buffer_length > left_tokens:
            chat_history.pop(0)
            curr_buffer_length = cls._get_num_tokens_from_messages(chat_history)
        full_inputs["chat_history"] = chat_history
        return full_inputs

    @staticmethod


        left = total_lines // 2 + 1
        right = total_lines - 1
        while left < right:
            mid = (left + right) >> 1
            text = "\n".join(lines[mid:])
            token = cls._count_tokens(text)
            if token > half_tokens:
                right = mid
            else:
                left = mid + 1
        second_half = "\n".join(lines[left:])

        if first_half != "" or second_half != "":
            return f"{first_half}\n...\n[too long to show]\n...\n{second_half}"
        else:
            # if len(first_half_list) == 0 and len(last_half_list) == 0:
            # if all lines >= max_token, return last 100 words as truncated results.
            return f"...\n[too long to show]\n...\n{raw_text[-100:]}"

    @classmethod
    def truncate_chat_history(cls, full_inputs: Dict[str, Any], max_token: int = 2500) -> Dict[str, Any]:
        _input = full_inputs["input"]
        agent_scratchpad = full_inputs["agent_scratchpad"]
        agent_scratchpad = "\n".join([_.content for _ in agent_scratchpad])
        _input_tokens = cls._count_tokens(_input)
        _scratchpad_tokens = cls._count_tokens(agent_scratchpad)

        left_tokens = max_token - _scratchpad_tokens - _input_tokens
        chat_history = full_inputs["chat_history"]

        curr_buffer_length = cls._get_num_tokens_from_messages(chat_history)
        while len(chat_history) != 0 and curr_buffer_length > left_tokens:
            chat_history.pop(0)
            curr_buffer_length = cls._get_num_tokens_from_messages(chat_history)
        full_inputs["chat_history"] = chat_history
        return full_inputs

    @staticmethod
    def _extract_value(json_string: str, key: str) -> str:
        pattern = re.compile(rf'"?{key}"?\s*:\s*("((?:[^"\\]|\\.)*)"|(\b[^,\s]*\b))', re.MULTILINE)
        match = pattern.search(json_string)
        if match:
            result = match.group(1).replace('\\"', '"').replace("\\\\", "\\").strip('"').strip("'").strip()
            # result = f"\"{result}\""
            return result
        raise ValueError(f"Could not find {key} in {json_string}")

    @staticmethod
    def _extract_response(
        chat_history: str,
        begin_marker: str = "[RESPONSE_BEGIN]",
        end_marker: str = "[RESPONSE_END]",
        ai_msg_marker: str = "AI:",
    ):
        code_blocks = chat_history.split(ai_msg_marker)
        pattern = r"\[RESPONSE_BEGIN\](.*?)\[RESPONSE_END\]"


    @classmethod
    def truncate_chat_history(cls, full_inputs: Dict[str, Any], max_token: int = 2500) -> Dict[str, Any]:
        _input = full_inputs["input"]
        agent_scratchpad = full_inputs["agent_scratchpad"]
        agent_scratchpad = "\n".join([_.content for _ in agent_scratchpad])
        _input_tokens = cls._count_tokens(_input)
        _scratchpad_tokens = cls._count_tokens(agent_scratchpad)

        left_tokens = max_token - _scratchpad_tokens - _input_tokens
        chat_history = full_inputs["chat_history"]

        curr_buffer_length = cls._get_num_tokens_from_messages(chat_history)
        while len(chat_history) != 0 and curr_buffer_length > left_tokens:
            chat_history.pop(0)
            curr_buffer_length = cls._get_num_tokens_from_messages(chat_history)
        full_inputs["chat_history"] = chat_history
        return full_inputs

    @staticmethod
    def _extract_value(json_string: str, key: str) -> str:
        pattern = re.compile(rf'"?{key}"?\s*:\s*("((?:[^"\\]|\\.)*)"|(\b[^,\s]*\b))', re.MULTILINE)
        match = pattern.search(json_string)
        if match:
            result = match.group(1).replace('\\"', '"').replace("\\\\", "\\").strip('"').strip("'").strip()
            # result = f"\"{result}\""
            return result
        raise ValueError(f"Could not find {key} in {json_string}")

    @staticmethod
    def _extract_response(
        chat_history: str,
        begin_marker: str = "[RESPONSE_BEGIN]",
        end_marker: str = "[RESPONSE_END]",
        ai_msg_marker: str = "AI:",
    ):
        code_blocks = chat_history.split(ai_msg_marker)
        pattern = r"\[RESPONSE_BEGIN\](.*?)\[RESPONSE_END\]"

        cleaned_output = []
        for code_block in code_blocks:
            matches = re.findall(pattern, code_block, re.DOTALL)
            if matches:
                cleaned_output.append(matches[0].strip())
        return "\n".join(cleaned_output)

    @classmethod
    def extract_action_for_llm(cls, text, max_token: int = 500) -> str:


        chat_history = full_inputs["chat_history"]

        curr_buffer_length = cls._get_num_tokens_from_messages(chat_history)
        while len(chat_history) != 0 and curr_buffer_length > left_tokens:
            chat_history.pop(0)
            curr_buffer_length = cls._get_num_tokens_from_messages(chat_history)
        full_inputs["chat_history"] = chat_history
        return full_inputs

    @staticmethod
    def _extract_value(json_string: str, key: str) -> str:
        pattern = re.compile(rf'"?{key}"?\s*:\s*("((?:[^"\\]|\\.)*)"|(\b[^,\s]*\b))', re.MULTILINE)
        match = pattern.search(json_string)
        if match:
            result = match.group(1).replace('\\"', '"').replace("\\\\", "\\").strip('"').strip("'").strip()
            # result = f"\"{result}\""
            return result
        raise ValueError(f"Could not find {key} in {json_string}")

    @staticmethod
    def _extract_response(
        chat_history: str,
        begin_marker: str = "[RESPONSE_BEGIN]",
        end_marker: str = "[RESPONSE_END]",
        ai_msg_marker: str = "AI:",
    ):
        code_blocks = chat_history.split(ai_msg_marker)
        pattern = r"\[RESPONSE_BEGIN\](.*?)\[RESPONSE_END\]"

        cleaned_output = []
        for code_block in code_blocks:
            matches = re.findall(pattern, code_block, re.DOTALL)
            if matches:
                cleaned_output.append(matches[0].strip())
        return "\n".join(cleaned_output)

    @classmethod
    def extract_action_for_llm(cls, text, max_token: int = 500) -> str:
        """Since Action should be fully inputted into an Agent, so we do not perform truncation here."""
        action_format = ACTION_FORMAT
        cleaned_output = text.strip()
        try:
            _action = cls._extract_value(cleaned_output, "action")
            _action_input = cls._extract_value(cleaned_output, "action_input")
            return action_format.format(_action=_action, _action_input=_action_input)
        except Exception:
            if cleaned_output.startswith("Action:"):
                lines = cleaned_output.splitlines()
                _action = lines[1].strip()
                _action_input = textwrap.dedent("\n".join(lines[2:])).strip()
                return action_format.format(_action=_action, _action_input=_action_input)
            else:
                _action_input = cleaned_output

            return action_format.format(_action="Final Answer", _action_input=_action_input)


        raise ValueError(f"Could not find {key} in {json_string}")

    @staticmethod
    def _extract_response(
        chat_history: str,
        begin_marker: str = "[RESPONSE_BEGIN]",
        end_marker: str = "[RESPONSE_END]",
        ai_msg_marker: str = "AI:",
    ):
        code_blocks = chat_history.split(ai_msg_marker)
        pattern = r"\[RESPONSE_BEGIN\](.*?)\[RESPONSE_END\]"

        cleaned_output = []
        for code_block in code_blocks:
            matches = re.findall(pattern, code_block, re.DOTALL)
            if matches:
                cleaned_output.append(matches[0].strip())
        return "\n".join(cleaned_output)

    @classmethod
    def extract_action_for_llm(cls, text, max_token: int = 500) -> str:
        """Since Action should be fully inputted into an Agent, so we do not perform truncation here."""
        action_format = ACTION_FORMAT
        cleaned_output = text.strip()
        try:
            _action = cls._extract_value(cleaned_output, "action")
            _action_input = cls._extract_value(cleaned_output, "action_input")
            return action_format.format(_action=_action, _action_input=_action_input)
        except Exception:
            if cleaned_output.startswith("Action:"):
                lines = cleaned_output.splitlines()
                _action = lines[1].strip()
                _action_input = textwrap.dedent("\n".join(lines[2:])).strip()
                return action_format.format(_action=_action, _action_input=_action_input)
            else:
                _action_input = cleaned_output

            return action_format.format(_action="Final Answer", _action_input=_action_input)

    @classmethod
    def extract_tool_response_for_llm(cls, text, tool_style: str = "code", max_token: int = 250) -> str:
        wrap_format = TOOL_RESPONSE_FORMAT
        tool_observation_format = TOOL_FORMAT[tool_style]
        cleaned_output = text.strip()
        if tool_style == "plugin":
            max_token = None

        try:
            _result = cls.truncate_text(cls._extract_value(cleaned_output, "result"), max_token)
            _intermediate_steps = cls.truncate_text(
                cls._extract_value(cleaned_output, "intermediate_steps"), max_token
            )
            _intermediate_steps = _intermediate_steps.replace("\\n", "\n").strip("\n")
            _result = _result.replace("\\n", "\n").strip("\n")
            _response = tool_observation_format.format(_intermediate_steps=_intermediate_steps, _result=_result)

            return wrap_format.format(_response=_response)
        except:


    def extract_action_for_llm(cls, text, max_token: int = 500) -> str:
        """Since Action should be fully inputted into an Agent, so we do not perform truncation here."""
        action_format = ACTION_FORMAT
        cleaned_output = text.strip()
        try:
            _action = cls._extract_value(cleaned_output, "action")
            _action_input = cls._extract_value(cleaned_output, "action_input")
            return action_format.format(_action=_action, _action_input=_action_input)
        except Exception:
            if cleaned_output.startswith("Action:"):
                lines = cleaned_output.splitlines()
                _action = lines[1].strip()
                _action_input = textwrap.dedent("\n".join(lines[2:])).strip()
                return action_format.format(_action=_action, _action_input=_action_input)
            else:
                _action_input = cleaned_output

            return action_format.format(_action="Final Answer", _action_input=_action_input)

    @classmethod
    def extract_tool_response_for_llm(cls, text, tool_style: str = "code", max_token: int = 250) -> str:
        wrap_format = TOOL_RESPONSE_FORMAT
        tool_observation_format = TOOL_FORMAT[tool_style]
        cleaned_output = text.strip()
        if tool_style == "plugin":
            max_token = None

        try:
            _result = cls.truncate_text(cls._extract_value(cleaned_output, "result"), max_token)
            _intermediate_steps = cls.truncate_text(
                cls._extract_value(cleaned_output, "intermediate_steps"), max_token
            )
            _intermediate_steps = _intermediate_steps.replace("\\n", "\n").strip("\n")
            _result = _result.replace("\\n", "\n").strip("\n")
            _response = tool_observation_format.format(_intermediate_steps=_intermediate_steps, _result=_result)

            return wrap_format.format(_response=_response)
        except:
            if cleaned_output.startswith("Final Answer:"):
                lines = cleaned_output.splitlines()
                _response = textwrap.dedent("\n".join(lines[2:])).strip()
                _response = cls.truncate_text(_response, max_token)
                return wrap_format.format(_response=_response)

            _response = cls.truncate_text(cleaned_output, max_token)
            return wrap_format.format(_response=_response)

    @classmethod
    def extract_code_for_python_tool(cls, text: str, max_token: int = 2500, trunc_ratio: float = 0.2) -> str:
        whole_code = MessageDataModel._extract_response(text)
        trunc_code = cls.truncate_text(whole_code, max_token=max_token, trunc_ratio=trunc_ratio)
        return trunc_code

    @classmethod
    def extract_code_for_sql_tool(cls, text: str, max_token: int = 2500, trunc_ratio: float = 0.2) -> str:
        whole_code = MessageDataModel._extract_response(text)
        trunc_code = cls.truncate_text(whole_code, max_token=max_token, trunc_ratio=trunc_ratio)
        return trunc_code


            _result = cls.truncate_text(cls._extract_value(cleaned_output, "result"), max_token)
            _intermediate_steps = cls.truncate_text(
                cls._extract_value(cleaned_output, "intermediate_steps"), max_token
            )
            _intermediate_steps = _intermediate_steps.replace("\\n", "\n").strip("\n")
            _result = _result.replace("\\n", "\n").strip("\n")
            _response = tool_observation_format.format(_intermediate_steps=_intermediate_steps, _result=_result)

            return wrap_format.format(_response=_response)
        except:
            if cleaned_output.startswith("Final Answer:"):
                lines = cleaned_output.splitlines()
                _response = textwrap.dedent("\n".join(lines[2:])).strip()
                _response = cls.truncate_text(_response, max_token)
                return wrap_format.format(_response=_response)

            _response = cls.truncate_text(cleaned_output, max_token)
            return wrap_format.format(_response=_response)

    @classmethod
    def extract_code_for_python_tool(cls, text: str, max_token: int = 2500, trunc_ratio: float = 0.2) -> str:
        whole_code = MessageDataModel._extract_response(text)
        trunc_code = cls.truncate_text(whole_code, max_token=max_token, trunc_ratio=trunc_ratio)
        return trunc_code

    @classmethod
    def extract_code_for_sql_tool(cls, text: str, max_token: int = 2500, trunc_ratio: float = 0.2) -> str:
        whole_code = MessageDataModel._extract_response(text)
        trunc_code = cls.truncate_text(whole_code, max_token=max_token, trunc_ratio=trunc_ratio)
        return trunc_code


            _response = tool_observation_format.format(_intermediate_steps=_intermediate_steps, _result=_result)

            return wrap_format.format(_response=_response)
        except:
            if cleaned_output.startswith("Final Answer:"):
                lines = cleaned_output.splitlines()
                _response = textwrap.dedent("\n".join(lines[2:])).strip()
                _response = cls.truncate_text(_response, max_token)
                return wrap_format.format(_response=_response)

            _response = cls.truncate_text(cleaned_output, max_token)
            return wrap_format.format(_response=_response)

    @classmethod
    def extract_code_for_python_tool(cls, text: str, max_token: int = 2500, trunc_ratio: float = 0.2) -> str:
        whole_code = MessageDataModel._extract_response(text)
        trunc_code = cls.truncate_text(whole_code, max_token=max_token, trunc_ratio=trunc_ratio)
        return trunc_code

    @classmethod
    def extract_code_for_sql_tool(cls, text: str, max_token: int = 2500, trunc_ratio: float = 0.2) -> str:
        whole_code = MessageDataModel._extract_response(text)
        trunc_code = cls.truncate_text(whole_code, max_token=max_token, trunc_ratio=trunc_ratio)
        return trunc_code


from __future__ import annotations

import uuid
from typing import Any

from pydantic import BaseModel

class DataModel(BaseModel):
    """Base class for data models."""

    id: str
    raw_data: Any
    raw_data_name: str
    raw_data_path: str
    llm_side_data: Any  # could be string or potentially images for future needs
    human_side_data: Any

    def __hash__(self) -> int:
        return hash(self.id)

    @classmethod
    def from_raw_data(
        cls, raw_data: Any, raw_data_name: str = "<default_name>", raw_data_path: str = "<default_path>", **kwargs: Any
    ) -> DataModel:
        uid = str(uuid.uuid4())
        return cls(id=uid, raw_data=raw_data, raw_data_name=raw_data_name, raw_data_path=raw_data_path, **kwargs)

    def get_id(self) -> str:
        return self.id

    def get_raw_data(self) -> Any:
        return self.raw_data

    def get_llm_side_data(self) -> Any:
        return self.raw_data

    def get_human_side_data(self) -> Any:
        return self.raw_data

    def __str__(self) -> str:
        return self.get_llm_side_data()


from __future__ import annotations

import uuid
from typing import Any

from pydantic import BaseModel

class DataModel(BaseModel):
    """Base class for data models."""

    id: str
    raw_data: Any
    raw_data_name: str
    raw_data_path: str
    llm_side_data: Any  # could be string or potentially images for future needs
    human_side_data: Any

    def __hash__(self) -> int:
        return hash(self.id)

    @classmethod
    def from_raw_data(
        cls, raw_data: Any, raw_data_name: str = "<default_name>", raw_data_path: str = "<default_path>", **kwargs: Any
    ) -> DataModel:
        uid = str(uuid.uuid4())
        return cls(id=uid, raw_data=raw_data, raw_data_name=raw_data_name, raw_data_path=raw_data_path, **kwargs)

    def get_id(self) -> str:
        return self.id

    def get_raw_data(self) -> Any:
        return self.raw_data

    def get_llm_side_data(self) -> Any:
        return self.raw_data

    def get_human_side_data(self) -> Any:
        return self.raw_data


import uuid
from typing import Any

from pydantic import BaseModel

class DataModel(BaseModel):
    """Base class for data models."""

    id: str
    raw_data: Any
    raw_data_name: str
    raw_data_path: str
    llm_side_data: Any  # could be string or potentially images for future needs
    human_side_data: Any

    def __hash__(self) -> int:
        return hash(self.id)

    @classmethod
    def from_raw_data(
        cls, raw_data: Any, raw_data_name: str = "<default_name>", raw_data_path: str = "<default_path>", **kwargs: Any
    ) -> DataModel:
        uid = str(uuid.uuid4())
        return cls(id=uid, raw_data=raw_data, raw_data_name=raw_data_name, raw_data_path=raw_data_path, **kwargs)

    def get_id(self) -> str:
        return self.id

    def get_raw_data(self) -> Any:
        return self.raw_data

    def get_llm_side_data(self) -> Any:
        return self.raw_data

    def get_human_side_data(self) -> Any:
        return self.raw_data

    def __str__(self) -> str:
        return self.get_llm_side_data()


class DataModel(BaseModel):
    """Base class for data models."""

    id: str
    raw_data: Any
    raw_data_name: str
    raw_data_path: str
    llm_side_data: Any  # could be string or potentially images for future needs
    human_side_data: Any

    def __hash__(self) -> int:
        return hash(self.id)

    @classmethod
    def from_raw_data(
        cls, raw_data: Any, raw_data_name: str = "<default_name>", raw_data_path: str = "<default_path>", **kwargs: Any
    ) -> DataModel:
        uid = str(uuid.uuid4())
        return cls(id=uid, raw_data=raw_data, raw_data_name=raw_data_name, raw_data_path=raw_data_path, **kwargs)

    def get_id(self) -> str:
        return self.id

    def get_raw_data(self) -> Any:
        return self.raw_data

    def get_llm_side_data(self) -> Any:
        return self.raw_data

    def get_human_side_data(self) -> Any:
        return self.raw_data

    def __str__(self) -> str:
        return self.get_llm_side_data()


    id: str
    raw_data: Any
    raw_data_name: str
    raw_data_path: str
    llm_side_data: Any  # could be string or potentially images for future needs
    human_side_data: Any

    def __hash__(self) -> int:
        return hash(self.id)

    @classmethod
    def from_raw_data(
        cls, raw_data: Any, raw_data_name: str = "<default_name>", raw_data_path: str = "<default_path>", **kwargs: Any
    ) -> DataModel:
        uid = str(uuid.uuid4())
        return cls(id=uid, raw_data=raw_data, raw_data_name=raw_data_name, raw_data_path=raw_data_path, **kwargs)

    def get_id(self) -> str:
        return self.id

    def get_raw_data(self) -> Any:
        return self.raw_data

    def get_llm_side_data(self) -> Any:
        return self.raw_data

    def get_human_side_data(self) -> Any:
        return self.raw_data

    def __str__(self) -> str:
        return self.get_llm_side_data()


    raw_data_path: str
    llm_side_data: Any  # could be string or potentially images for future needs
    human_side_data: Any

    def __hash__(self) -> int:
        return hash(self.id)

    @classmethod
    def from_raw_data(
        cls, raw_data: Any, raw_data_name: str = "<default_name>", raw_data_path: str = "<default_path>", **kwargs: Any
    ) -> DataModel:
        uid = str(uuid.uuid4())
        return cls(id=uid, raw_data=raw_data, raw_data_name=raw_data_name, raw_data_path=raw_data_path, **kwargs)

    def get_id(self) -> str:
        return self.id

    def get_raw_data(self) -> Any:
        return self.raw_data

    def get_llm_side_data(self) -> Any:
        return self.raw_data

    def get_human_side_data(self) -> Any:
        return self.raw_data

    def __str__(self) -> str:
        return self.get_llm_side_data()


    def __hash__(self) -> int:
        return hash(self.id)

    @classmethod
    def from_raw_data(
        cls, raw_data: Any, raw_data_name: str = "<default_name>", raw_data_path: str = "<default_path>", **kwargs: Any
    ) -> DataModel:
        uid = str(uuid.uuid4())
        return cls(id=uid, raw_data=raw_data, raw_data_name=raw_data_name, raw_data_path=raw_data_path, **kwargs)

    def get_id(self) -> str:
        return self.id

    def get_raw_data(self) -> Any:
        return self.raw_data

    def get_llm_side_data(self) -> Any:
        return self.raw_data

    def get_human_side_data(self) -> Any:
        return self.raw_data

    def __str__(self) -> str:
        return self.get_llm_side_data()


    @classmethod
    def from_raw_data(
        cls, raw_data: Any, raw_data_name: str = "<default_name>", raw_data_path: str = "<default_path>", **kwargs: Any
    ) -> DataModel:
        uid = str(uuid.uuid4())
        return cls(id=uid, raw_data=raw_data, raw_data_name=raw_data_name, raw_data_path=raw_data_path, **kwargs)

    def get_id(self) -> str:
        return self.id

    def get_raw_data(self) -> Any:
        return self.raw_data

    def get_llm_side_data(self) -> Any:
        return self.raw_data

    def get_human_side_data(self) -> Any:
        return self.raw_data

    def __str__(self) -> str:
        return self.get_llm_side_data()


import json
from bs4 import BeautifulSoup
from collections import defaultdict
from typing import Any, Dict, List, Union
from real_agents.adapters.data_model.base import DataModel
import requests
import re
import tiktoken

JsonNode = Dict[str, Union[str, List[Any], int]]
PossibleTemplate = Dict[str, Union[str, List[Any], int]]
OptimizedTemplate = Dict[str, Union[str, List[Any], int, set]]
PossibleTemplates = Dict[str, PossibleTemplate]

def find_potential_templates(node, possible_templates):
    """Find all potential templates in the HTML tree."""
    if node.name:  # Element node
        attributes = {attr: node[attr] for attr in node.attrs}
        children = []
        for child in node.children:
            child_json = find_potential_templates(child, possible_templates)
            if child_json:
                children.append(child_json)

        # Max depth of the tree
        depth = max([c["depth"] for c in children], default=0) + 1

        # Create a template hash
        template_hash = f"{node.name}#{sorted(attributes.keys())}#{[c['template_hash'] for c in children]}"

        # Gather template values
        template_values = list(attributes.values()) + [val for c in children for val in c["template_values"]]

        json_node = {
            "type": "ELEMENT",
            "tag_name": node.name,
            "attributes": attributes,
            "children": children,
            "template_hash": template_hash,
            "template_values": template_values,
            "depth": depth,
        }

        # Add node to possible templates
        if template_hash in possible_templates:
            if possible_templates[template_hash][0]["depth"] != depth:
                raise ValueError(f"Template depth mismatch for template {template_hash}")
            possible_templates[template_hash].append(json_node)
        else:
            possible_templates[template_hash] = [json_node]

        return json_node
    elif isinstance(node, str):  # Text node
        text = node.strip()
        if text:
            return {"type": "TEXT", "content": text, "template_hash": "TEXT", "template_values": [text], "depth": 0}
    return None

def optimize_template(template):
    """Check and adjust the template in possible_templates to optimize style."""
    values_to_inline = {
        i
        for i in range(len(template["nodes"][0]["templateValues"]))
        if all(n["templateValues"][i] == template["nodes"][0]["templateValues"][i] for n in template["nodes"])
    }
    return {**template, "valuesToInline": values_to_inline}

def is_string_a_number(s):
    try:
        float(s)
        return True
    except ValueError:
        return False

def get_placeholder(template, value_index):
    """Get the placeholder for the value at the given index in the template."""
    placeholder_index = value_index + 1 - len([i for i in template["valuesToInline"] if i < value_index])
    return f"${placeholder_index}"

def create_template_tree(node, templates, render_for_template, current_value_index=0):
    """Convert the DOM into processed template tree."""
    if node["type"] == "TEXT":
        if current_value_index in render_for_template["valuesToInline"]:
            return {
                "template": node["content"],
                "valueIndex": current_value_index + 1,
                "consumedTemplates": [node["templateHash"]],
            }
        else:
            return {
                "template": get_placeholder(render_for_template, current_value_index),
                "valueIndex": current_value_index + 1,
                "consumedTemplates": [node["templateHash"]],
            }

    else:
        updated_value_index = current_value_index
        consumed_templates = [node["templateHash"]]

        attrs = "".join(
            [
                f' {k}="{v}"'
                if updated_value_index + i in render_for_template["valuesToInline"]
                else f" {k}={get_placeholder(render_for_template, updated_value_index + i)}"
                for i, (k, v) in enumerate(node["attributes"].items())
            ]
        )
        updated_value_index += len(node["attributes"])

        children = []
        for child in node["children"]:
            child_template = create_template_tree(child, templates, render_for_template, updated_value_index)
            children.append(child_template["template"])
            updated_value_index = child_template["valueIndex"]
            consumed_templates.extend(child_template["consumedTemplates"])

        return {
            "template": f"<{node['tagName'].lower()}{attrs}/>"
            if not children
            else f"<{node['tagName'].lower()}{attrs}>{''.join(children)}</{node['tagName'].lower()}>",
            "valueIndex": updated_value_index,
            "consumedTemplates": consumed_templates,
        }

def serialize_tree(node, templates):
    """Serialize the template tree into HTML string."""
    if node["type"] == "TEXT":
        return node["content"]
    elif node["templateHash"] in templates:
        template = templates[node["templateHash"]]
        return f"{{T{template['label']}({','.join([str(v) if is_string_a_number(v) else json.dumps(v) for i, v in enumerate(node['templateValues']) if i not in template['valuesToInline']])})}}"
    else:
        attrs = "".join([f' {k}="{v}"' for k, v in node["attributes"].items()])
        children = "".join([serialize_tree(c, templates) for c in node["children"]])
        return (
            f"<{node['tagName'].lower()}{attrs}/>"
            if not children
            else f"<{node['tagName'].lower()}{attrs}>{children}</{node['tagName'].lower()}>"
        )

def truncate_html_by_tokens(html_string, max_tokens, model_name, num_tags_to_remove_each_time=10):
    tokens_count = count_tokens(html_string, model_name)
    num_tags_to_remove_each_time = round(tokens_count / 500)
    soup = BeautifulSoup(html_string, "html.parser")
    # Remove all iframe tags
    html_string = remove_iframes(html_string)
    while tokens_count > max_tokens:
        tags = soup.find_all(True)  # find all tags
        # remove the last N tags
        for tag in tags[-num_tags_to_remove_each_time:]:
            tag.decompose()

        html_string = str(soup)

        # re-count the tokens
        tokens_count = count_tokens(html_string, model_name)

    return html_string

# hacky way
def remove_iframes(html_string):
    # Remove all iframe tags using regex
    return re.sub("<iframe.*?/iframe>", "", html_string, flags=re.DOTALL)

# if you wanna change encoding schema, refer to https://github.com/openai/openai-cookbook/blob/main/examples/How_to_count_tokens_with_tiktoken.ipynb
def count_tokens(text, model_name):
    encoding = tiktoken.get_encoding("cl100k_base")
    return len(encoding.encode(text))

class HTMLDataModel(DataModel):
    """A data model for HTML, for webot purpose."""

    def get_llm_side_data(self) -> str:
        html_string = self.raw_data
        truncated_html_string = truncate_html_by_tokens(html_string, 5000, "gpt-4")
        return truncated_html_string


import json
from bs4 import BeautifulSoup
from collections import defaultdict
from typing import Any, Dict, List, Union
from real_agents.adapters.data_model.base import DataModel
import requests
import re
import tiktoken

JsonNode = Dict[str, Union[str, List[Any], int]]
PossibleTemplate = Dict[str, Union[str, List[Any], int]]
OptimizedTemplate = Dict[str, Union[str, List[Any], int, set]]
PossibleTemplates = Dict[str, PossibleTemplate]

def find_potential_templates(node, possible_templates):
    """Find all potential templates in the HTML tree."""
    if node.name:  # Element node
        attributes = {attr: node[attr] for attr in node.attrs}
        children = []
        for child in node.children:
            child_json = find_potential_templates(child, possible_templates)
            if child_json:
                children.append(child_json)

        # Max depth of the tree
        depth = max([c["depth"] for c in children], default=0) + 1

        # Create a template hash
        template_hash = f"{node.name}#{sorted(attributes.keys())}#{[c['template_hash'] for c in children]}"

        # Gather template values
        template_values = list(attributes.values()) + [val for c in children for val in c["template_values"]]

        json_node = {
            "type": "ELEMENT",
            "tag_name": node.name,
            "attributes": attributes,
            "children": children,
            "template_hash": template_hash,
            "template_values": template_values,
            "depth": depth,
        }

        # Add node to possible templates
        if template_hash in possible_templates:
            if possible_templates[template_hash][0]["depth"] != depth:
                raise ValueError(f"Template depth mismatch for template {template_hash}")
            possible_templates[template_hash].append(json_node)
        else:
            possible_templates[template_hash] = [json_node]

        return json_node
    elif isinstance(node, str):  # Text node
        text = node.strip()
        if text:
            return {"type": "TEXT", "content": text, "template_hash": "TEXT", "template_values": [text], "depth": 0}
    return None

def optimize_template(template):
    """Check and adjust the template in possible_templates to optimize style."""
    values_to_inline = {
        i
        for i in range(len(template["nodes"][0]["templateValues"]))
        if all(n["templateValues"][i] == template["nodes"][0]["templateValues"][i] for n in template["nodes"])
    }
    return {**template, "valuesToInline": values_to_inline}

def is_string_a_number(s):
    try:
        float(s)
        return True
    except ValueError:
        return False


            "template_values": template_values,
            "depth": depth,
        }

        # Add node to possible templates
        if template_hash in possible_templates:
            if possible_templates[template_hash][0]["depth"] != depth:
                raise ValueError(f"Template depth mismatch for template {template_hash}")
            possible_templates[template_hash].append(json_node)
        else:
            possible_templates[template_hash] = [json_node]

        return json_node
    elif isinstance(node, str):  # Text node
        text = node.strip()
        if text:
            return {"type": "TEXT", "content": text, "template_hash": "TEXT", "template_values": [text], "depth": 0}
    return None

def optimize_template(template):
    """Check and adjust the template in possible_templates to optimize style."""
    values_to_inline = {
        i
        for i in range(len(template["nodes"][0]["templateValues"]))
        if all(n["templateValues"][i] == template["nodes"][0]["templateValues"][i] for n in template["nodes"])
    }
    return {**template, "valuesToInline": values_to_inline}

def is_string_a_number(s):
    try:
        float(s)
        return True
    except ValueError:
        return False

def get_placeholder(template, value_index):
    """Get the placeholder for the value at the given index in the template."""
    placeholder_index = value_index + 1 - len([i for i in template["valuesToInline"] if i < value_index])
    return f"${placeholder_index}"

def create_template_tree(node, templates, render_for_template, current_value_index=0):
    """Convert the DOM into processed template tree."""
    if node["type"] == "TEXT":
        if current_value_index in render_for_template["valuesToInline"]:


            possible_templates[template_hash] = [json_node]

        return json_node
    elif isinstance(node, str):  # Text node
        text = node.strip()
        if text:
            return {"type": "TEXT", "content": text, "template_hash": "TEXT", "template_values": [text], "depth": 0}
    return None

def optimize_template(template):
    """Check and adjust the template in possible_templates to optimize style."""
    values_to_inline = {
        i
        for i in range(len(template["nodes"][0]["templateValues"]))
        if all(n["templateValues"][i] == template["nodes"][0]["templateValues"][i] for n in template["nodes"])
    }
    return {**template, "valuesToInline": values_to_inline}

def is_string_a_number(s):
    try:
        float(s)
        return True
    except ValueError:
        return False

def get_placeholder(template, value_index):
    """Get the placeholder for the value at the given index in the template."""
    placeholder_index = value_index + 1 - len([i for i in template["valuesToInline"] if i < value_index])
    return f"${placeholder_index}"

def create_template_tree(node, templates, render_for_template, current_value_index=0):
    """Convert the DOM into processed template tree."""
    if node["type"] == "TEXT":
        if current_value_index in render_for_template["valuesToInline"]:
            return {
                "template": node["content"],
                "valueIndex": current_value_index + 1,
                "consumedTemplates": [node["templateHash"]],
            }
        else:
            return {
                "template": get_placeholder(render_for_template, current_value_index),


def optimize_template(template):
    """Check and adjust the template in possible_templates to optimize style."""
    values_to_inline = {
        i
        for i in range(len(template["nodes"][0]["templateValues"]))
        if all(n["templateValues"][i] == template["nodes"][0]["templateValues"][i] for n in template["nodes"])
    }
    return {**template, "valuesToInline": values_to_inline}

def is_string_a_number(s):
    try:
        float(s)
        return True
    except ValueError:
        return False

def get_placeholder(template, value_index):
    """Get the placeholder for the value at the given index in the template."""
    placeholder_index = value_index + 1 - len([i for i in template["valuesToInline"] if i < value_index])
    return f"${placeholder_index}"

def create_template_tree(node, templates, render_for_template, current_value_index=0):
    """Convert the DOM into processed template tree."""
    if node["type"] == "TEXT":
        if current_value_index in render_for_template["valuesToInline"]:
            return {
                "template": node["content"],
                "valueIndex": current_value_index + 1,
                "consumedTemplates": [node["templateHash"]],
            }
        else:
            return {
                "template": get_placeholder(render_for_template, current_value_index),
                "valueIndex": current_value_index + 1,
                "consumedTemplates": [node["templateHash"]],
            }

    else:
        updated_value_index = current_value_index


        for i in range(len(template["nodes"][0]["templateValues"]))
        if all(n["templateValues"][i] == template["nodes"][0]["templateValues"][i] for n in template["nodes"])
    }
    return {**template, "valuesToInline": values_to_inline}

def is_string_a_number(s):
    try:
        float(s)
        return True
    except ValueError:
        return False

def get_placeholder(template, value_index):
    """Get the placeholder for the value at the given index in the template."""
    placeholder_index = value_index + 1 - len([i for i in template["valuesToInline"] if i < value_index])
    return f"${placeholder_index}"

def create_template_tree(node, templates, render_for_template, current_value_index=0):
    """Convert the DOM into processed template tree."""
    if node["type"] == "TEXT":
        if current_value_index in render_for_template["valuesToInline"]:
            return {
                "template": node["content"],
                "valueIndex": current_value_index + 1,
                "consumedTemplates": [node["templateHash"]],
            }
        else:
            return {
                "template": get_placeholder(render_for_template, current_value_index),
                "valueIndex": current_value_index + 1,
                "consumedTemplates": [node["templateHash"]],
            }

    else:
        updated_value_index = current_value_index
        consumed_templates = [node["templateHash"]]

        attrs = "".join(
            [
                f' {k}="{v}"'
                if updated_value_index + i in render_for_template["valuesToInline"]
                else f" {k}={get_placeholder(render_for_template, updated_value_index + i)}"
                for i, (k, v) in enumerate(node["attributes"].items())
            ]
        )
        updated_value_index += len(node["attributes"])

        children = []
        for child in node["children"]:
            child_template = create_template_tree(child, templates, render_for_template, updated_value_index)
            children.append(child_template["template"])
            updated_value_index = child_template["valueIndex"]
            consumed_templates.extend(child_template["consumedTemplates"])

        return {
            "template": f"<{node['tagName'].lower()}{attrs}/>"
            if not children
            else f"<{node['tagName'].lower()}{attrs}>{''.join(children)}</{node['tagName'].lower()}>",
            "valueIndex": updated_value_index,
            "consumedTemplates": consumed_templates,
        }

def serialize_tree(node, templates):
    """Serialize the template tree into HTML string."""
    if node["type"] == "TEXT":
        return node["content"]
    elif node["templateHash"] in templates:
        template = templates[node["templateHash"]]
        return f"{{T{template['label']}({','.join([str(v) if is_string_a_number(v) else json.dumps(v) for i, v in enumerate(node['templateValues']) if i not in template['valuesToInline']])})}}"
    else:
        attrs = "".join([f' {k}="{v}"' for k, v in node["attributes"].items()])
        children = "".join([serialize_tree(c, templates) for c in node["children"]])
        return (
            f"<{node['tagName'].lower()}{attrs}/>"
            if not children
            else f"<{node['tagName'].lower()}{attrs}>{children}</{node['tagName'].lower()}>"
        )

def truncate_html_by_tokens(html_string, max_tokens, model_name, num_tags_to_remove_each_time=10):


            ]
        )
        updated_value_index += len(node["attributes"])

        children = []
        for child in node["children"]:
            child_template = create_template_tree(child, templates, render_for_template, updated_value_index)
            children.append(child_template["template"])
            updated_value_index = child_template["valueIndex"]
            consumed_templates.extend(child_template["consumedTemplates"])

        return {
            "template": f"<{node['tagName'].lower()}{attrs}/>"
            if not children
            else f"<{node['tagName'].lower()}{attrs}>{''.join(children)}</{node['tagName'].lower()}>",
            "valueIndex": updated_value_index,
            "consumedTemplates": consumed_templates,
        }

def serialize_tree(node, templates):
    """Serialize the template tree into HTML string."""
    if node["type"] == "TEXT":
        return node["content"]
    elif node["templateHash"] in templates:
        template = templates[node["templateHash"]]
        return f"{{T{template['label']}({','.join([str(v) if is_string_a_number(v) else json.dumps(v) for i, v in enumerate(node['templateValues']) if i not in template['valuesToInline']])})}}"
    else:
        attrs = "".join([f' {k}="{v}"' for k, v in node["attributes"].items()])
        children = "".join([serialize_tree(c, templates) for c in node["children"]])
        return (
            f"<{node['tagName'].lower()}{attrs}/>"
            if not children
            else f"<{node['tagName'].lower()}{attrs}>{children}</{node['tagName'].lower()}>"
        )

def truncate_html_by_tokens(html_string, max_tokens, model_name, num_tags_to_remove_each_time=10):
    tokens_count = count_tokens(html_string, model_name)
    num_tags_to_remove_each_time = round(tokens_count / 500)
    soup = BeautifulSoup(html_string, "html.parser")
    # Remove all iframe tags
    html_string = remove_iframes(html_string)
    while tokens_count > max_tokens:
        tags = soup.find_all(True)  # find all tags
        # remove the last N tags
        for tag in tags[-num_tags_to_remove_each_time:]:
            tag.decompose()

        html_string = str(soup)

        # re-count the tokens
        tokens_count = count_tokens(html_string, model_name)

    return html_string


        }

def serialize_tree(node, templates):
    """Serialize the template tree into HTML string."""
    if node["type"] == "TEXT":
        return node["content"]
    elif node["templateHash"] in templates:
        template = templates[node["templateHash"]]
        return f"{{T{template['label']}({','.join([str(v) if is_string_a_number(v) else json.dumps(v) for i, v in enumerate(node['templateValues']) if i not in template['valuesToInline']])})}}"
    else:
        attrs = "".join([f' {k}="{v}"' for k, v in node["attributes"].items()])
        children = "".join([serialize_tree(c, templates) for c in node["children"]])
        return (
            f"<{node['tagName'].lower()}{attrs}/>"
            if not children
            else f"<{node['tagName'].lower()}{attrs}>{children}</{node['tagName'].lower()}>"
        )

def truncate_html_by_tokens(html_string, max_tokens, model_name, num_tags_to_remove_each_time=10):
    tokens_count = count_tokens(html_string, model_name)
    num_tags_to_remove_each_time = round(tokens_count / 500)
    soup = BeautifulSoup(html_string, "html.parser")
    # Remove all iframe tags
    html_string = remove_iframes(html_string)
    while tokens_count > max_tokens:
        tags = soup.find_all(True)  # find all tags
        # remove the last N tags
        for tag in tags[-num_tags_to_remove_each_time:]:
            tag.decompose()

        html_string = str(soup)

        # re-count the tokens
        tokens_count = count_tokens(html_string, model_name)

    return html_string

# hacky way
def remove_iframes(html_string):
    # Remove all iframe tags using regex
    return re.sub("<iframe.*?/iframe>", "", html_string, flags=re.DOTALL)

# if you wanna change encoding schema, refer to https://github.com/openai/openai-cookbook/blob/main/examples/How_to_count_tokens_with_tiktoken.ipynb
def count_tokens(text, model_name):
    encoding = tiktoken.get_encoding("cl100k_base")
    return len(encoding.encode(text))

class HTMLDataModel(DataModel):
    """A data model for HTML, for webot purpose."""

    def get_llm_side_data(self) -> str:
        html_string = self.raw_data
        truncated_html_string = truncate_html_by_tokens(html_string, 5000, "gpt-4")


    tokens_count = count_tokens(html_string, model_name)
    num_tags_to_remove_each_time = round(tokens_count / 500)
    soup = BeautifulSoup(html_string, "html.parser")
    # Remove all iframe tags
    html_string = remove_iframes(html_string)
    while tokens_count > max_tokens:
        tags = soup.find_all(True)  # find all tags
        # remove the last N tags
        for tag in tags[-num_tags_to_remove_each_time:]:
            tag.decompose()

        html_string = str(soup)

        # re-count the tokens
        tokens_count = count_tokens(html_string, model_name)

    return html_string

# hacky way
def remove_iframes(html_string):
    # Remove all iframe tags using regex
    return re.sub("<iframe.*?/iframe>", "", html_string, flags=re.DOTALL)

# if you wanna change encoding schema, refer to https://github.com/openai/openai-cookbook/blob/main/examples/How_to_count_tokens_with_tiktoken.ipynb
def count_tokens(text, model_name):
    encoding = tiktoken.get_encoding("cl100k_base")
    return len(encoding.encode(text))

class HTMLDataModel(DataModel):
    """A data model for HTML, for webot purpose."""

    def get_llm_side_data(self) -> str:
        html_string = self.raw_data
        truncated_html_string = truncate_html_by_tokens(html_string, 5000, "gpt-4")
        return truncated_html_string


        tags = soup.find_all(True)  # find all tags
        # remove the last N tags
        for tag in tags[-num_tags_to_remove_each_time:]:
            tag.decompose()

        html_string = str(soup)

        # re-count the tokens
        tokens_count = count_tokens(html_string, model_name)

    return html_string

# hacky way
def remove_iframes(html_string):
    # Remove all iframe tags using regex
    return re.sub("<iframe.*?/iframe>", "", html_string, flags=re.DOTALL)

# if you wanna change encoding schema, refer to https://github.com/openai/openai-cookbook/blob/main/examples/How_to_count_tokens_with_tiktoken.ipynb
def count_tokens(text, model_name):
    encoding = tiktoken.get_encoding("cl100k_base")
    return len(encoding.encode(text))

class HTMLDataModel(DataModel):
    """A data model for HTML, for webot purpose."""

    def get_llm_side_data(self) -> str:
        html_string = self.raw_data
        truncated_html_string = truncate_html_by_tokens(html_string, 5000, "gpt-4")
        return truncated_html_string


HTMLDataModel

        html_string = str(soup)

        # re-count the tokens
        tokens_count = count_tokens(html_string, model_name)

    return html_string

# hacky way
def remove_iframes(html_string):
    # Remove all iframe tags using regex
    return re.sub("<iframe.*?/iframe>", "", html_string, flags=re.DOTALL)

# if you wanna change encoding schema, refer to https://github.com/openai/openai-cookbook/blob/main/examples/How_to_count_tokens_with_tiktoken.ipynb
def count_tokens(text, model_name):
    encoding = tiktoken.get_encoding("cl100k_base")
    return len(encoding.encode(text))

class HTMLDataModel(DataModel):
    """A data model for HTML, for webot purpose."""

    def get_llm_side_data(self) -> str:
        html_string = self.raw_data
        truncated_html_string = truncate_html_by_tokens(html_string, 5000, "gpt-4")
        return truncated_html_string


        tokens_count = count_tokens(html_string, model_name)

    return html_string

# hacky way
def remove_iframes(html_string):
    # Remove all iframe tags using regex
    return re.sub("<iframe.*?/iframe>", "", html_string, flags=re.DOTALL)

# if you wanna change encoding schema, refer to https://github.com/openai/openai-cookbook/blob/main/examples/How_to_count_tokens_with_tiktoken.ipynb
def count_tokens(text, model_name):
    encoding = tiktoken.get_encoding("cl100k_base")
    return len(encoding.encode(text))

class HTMLDataModel(DataModel):
    """A data model for HTML, for webot purpose."""

    def get_llm_side_data(self) -> str:
        html_string = self.raw_data
        truncated_html_string = truncate_html_by_tokens(html_string, 5000, "gpt-4")
        return truncated_html_string


from __future__ import annotations

import os
from typing import Any

import pandas as pd
from sqlalchemy import create_engine, inspect

from real_agents.adapters.data_model.base import DataModel
from real_agents.adapters.data_model.table import TableDataModel
from real_agents.adapters.data_model.templates.skg_templates.database_templates import serialize_db
from real_agents.adapters.schema import SQLDatabase

class DatabaseDataModel(DataModel):
    """A data model for database."""

    @classmethod
    def from_table_data_model(cls, table_data_model: TableDataModel) -> DatabaseDataModel:
        os.makedirs(f".db_cache/{table_data_model.id}", exist_ok=True)
        db_path = os.path.join(
            f".db_cache/{table_data_model.id}", os.path.splitext(table_data_model.raw_data_name)[0] + ".db"
        )
        engine = create_engine(f"sqlite:///{db_path}")
        table_data_model.raw_data.to_sql(table_data_model.raw_data_name, engine, if_exists="replace")
        db = SQLDatabase(engine)
        return cls.from_raw_data(raw_data=db, raw_data_name=table_data_model.raw_data_name)

    def insert_table_data_model(self, table_data_model: TableDataModel) -> None:
        engine = self.raw_data.engine
        table_data_model.raw_data.to_sql(table_data_model.raw_data_name, engine)

    def get_llm_side_data(self, serialize_method: str = "database", num_visible_rows: int = 3) -> Any:
        db = self.raw_data
        formatted_db = serialize_db(db, serialize_method, num_visible_rows)
        return formatted_db

    def get_human_side_data(self) -> Any:
        # In the frontend, we show the first few rows of each table
        engine = self.raw_data.engine
        inspector = inspect(engine)
        table_names = inspector.get_table_names()

        # Loop through each table name, creating a DataFrame from the first three rows of each table
        df_dict = {}
        for table_name in table_names:
            query = f"SELECT * FROM {table_name} LIMIT 3"
            df = pd.read_sql(query, engine)
            df_dict[table_name] = df
        return df_dict


from __future__ import annotations

import os
from typing import Any

import pandas as pd
from sqlalchemy import create_engine, inspect

from real_agents.adapters.data_model.base import DataModel
from real_agents.adapters.data_model.table import TableDataModel
from real_agents.adapters.data_model.templates.skg_templates.database_templates import serialize_db
from real_agents.adapters.schema import SQLDatabase

class DatabaseDataModel(DataModel):
    """A data model for database."""

    @classmethod
    def from_table_data_model(cls, table_data_model: TableDataModel) -> DatabaseDataModel:
        os.makedirs(f".db_cache/{table_data_model.id}", exist_ok=True)
        db_path = os.path.join(
            f".db_cache/{table_data_model.id}", os.path.splitext(table_data_model.raw_data_name)[0] + ".db"
        )
        engine = create_engine(f"sqlite:///{db_path}")
        table_data_model.raw_data.to_sql(table_data_model.raw_data_name, engine, if_exists="replace")
        db = SQLDatabase(engine)
        return cls.from_raw_data(raw_data=db, raw_data_name=table_data_model.raw_data_name)

    def insert_table_data_model(self, table_data_model: TableDataModel) -> None:
        engine = self.raw_data.engine
        table_data_model.raw_data.to_sql(table_data_model.raw_data_name, engine)

    def get_llm_side_data(self, serialize_method: str = "database", num_visible_rows: int = 3) -> Any:
        db = self.raw_data
        formatted_db = serialize_db(db, serialize_method, num_visible_rows)
        return formatted_db

    def get_human_side_data(self) -> Any:
        # In the frontend, we show the first few rows of each table
        engine = self.raw_data.engine
        inspector = inspect(engine)
        table_names = inspector.get_table_names()

        # Loop through each table name, creating a DataFrame from the first three rows of each table
        df_dict = {}
        for table_name in table_names:
            query = f"SELECT * FROM {table_name} LIMIT 3"


from real_agents.adapters.data_model.base import DataModel
from real_agents.adapters.data_model.table import TableDataModel
from real_agents.adapters.data_model.templates.skg_templates.database_templates import serialize_db
from real_agents.adapters.schema import SQLDatabase

class DatabaseDataModel(DataModel):
    """A data model for database."""

    @classmethod
    def from_table_data_model(cls, table_data_model: TableDataModel) -> DatabaseDataModel:
        os.makedirs(f".db_cache/{table_data_model.id}", exist_ok=True)
        db_path = os.path.join(
            f".db_cache/{table_data_model.id}", os.path.splitext(table_data_model.raw_data_name)[0] + ".db"
        )
        engine = create_engine(f"sqlite:///{db_path}")
        table_data_model.raw_data.to_sql(table_data_model.raw_data_name, engine, if_exists="replace")
        db = SQLDatabase(engine)
        return cls.from_raw_data(raw_data=db, raw_data_name=table_data_model.raw_data_name)

    def insert_table_data_model(self, table_data_model: TableDataModel) -> None:
        engine = self.raw_data.engine
        table_data_model.raw_data.to_sql(table_data_model.raw_data_name, engine)

    def get_llm_side_data(self, serialize_method: str = "database", num_visible_rows: int = 3) -> Any:
        db = self.raw_data
        formatted_db = serialize_db(db, serialize_method, num_visible_rows)
        return formatted_db

    def get_human_side_data(self) -> Any:
        # In the frontend, we show the first few rows of each table
        engine = self.raw_data.engine
        inspector = inspect(engine)
        table_names = inspector.get_table_names()

        # Loop through each table name, creating a DataFrame from the first three rows of each table
        df_dict = {}
        for table_name in table_names:
            query = f"SELECT * FROM {table_name} LIMIT 3"
            df = pd.read_sql(query, engine)
            df_dict[table_name] = df
        return df_dict


class DatabaseDataModel(DataModel):
    """A data model for database."""

    @classmethod
    def from_table_data_model(cls, table_data_model: TableDataModel) -> DatabaseDataModel:
        os.makedirs(f".db_cache/{table_data_model.id}", exist_ok=True)
        db_path = os.path.join(
            f".db_cache/{table_data_model.id}", os.path.splitext(table_data_model.raw_data_name)[0] + ".db"
        )
        engine = create_engine(f"sqlite:///{db_path}")
        table_data_model.raw_data.to_sql(table_data_model.raw_data_name, engine, if_exists="replace")
        db = SQLDatabase(engine)
        return cls.from_raw_data(raw_data=db, raw_data_name=table_data_model.raw_data_name)

    def insert_table_data_model(self, table_data_model: TableDataModel) -> None:
        engine = self.raw_data.engine
        table_data_model.raw_data.to_sql(table_data_model.raw_data_name, engine)

    def get_llm_side_data(self, serialize_method: str = "database", num_visible_rows: int = 3) -> Any:
        db = self.raw_data
        formatted_db = serialize_db(db, serialize_method, num_visible_rows)
        return formatted_db

    def get_human_side_data(self) -> Any:
        # In the frontend, we show the first few rows of each table
        engine = self.raw_data.engine
        inspector = inspect(engine)
        table_names = inspector.get_table_names()

        # Loop through each table name, creating a DataFrame from the first three rows of each table
        df_dict = {}
        for table_name in table_names:
            query = f"SELECT * FROM {table_name} LIMIT 3"
            df = pd.read_sql(query, engine)
            df_dict[table_name] = df
        return df_dict


    @classmethod
    def from_table_data_model(cls, table_data_model: TableDataModel) -> DatabaseDataModel:
        os.makedirs(f".db_cache/{table_data_model.id}", exist_ok=True)
        db_path = os.path.join(
            f".db_cache/{table_data_model.id}", os.path.splitext(table_data_model.raw_data_name)[0] + ".db"
        )
        engine = create_engine(f"sqlite:///{db_path}")
        table_data_model.raw_data.to_sql(table_data_model.raw_data_name, engine, if_exists="replace")
        db = SQLDatabase(engine)
        return cls.from_raw_data(raw_data=db, raw_data_name=table_data_model.raw_data_name)

    def insert_table_data_model(self, table_data_model: TableDataModel) -> None:
        engine = self.raw_data.engine
        table_data_model.raw_data.to_sql(table_data_model.raw_data_name, engine)

    def get_llm_side_data(self, serialize_method: str = "database", num_visible_rows: int = 3) -> Any:
        db = self.raw_data
        formatted_db = serialize_db(db, serialize_method, num_visible_rows)
        return formatted_db

    def get_human_side_data(self) -> Any:
        # In the frontend, we show the first few rows of each table
        engine = self.raw_data.engine
        inspector = inspect(engine)
        table_names = inspector.get_table_names()

        # Loop through each table name, creating a DataFrame from the first three rows of each table
        df_dict = {}
        for table_name in table_names:
            query = f"SELECT * FROM {table_name} LIMIT 3"
            df = pd.read_sql(query, engine)
            df_dict[table_name] = df
        return df_dict


def indent_multiline_string(multiline_string: str, indent: int = 1) -> str:
    return "\n".join("\t" * indent + line for line in multiline_string.split("\n"))


from real_agents.adapters.data_model.base import DataModel

class TextDataModel(DataModel):
    """A data model for text, general purpose."""

    def get_llm_side_data(self, max_chars: int = 5000) -> str:
        assert isinstance(self.raw_data, str)
        return self.raw_data[:max_chars]


from __future__ import annotations

import json
from typing import Any

from pandas import DataFrame

from real_agents.adapters.data_model.base import DataModel
from real_agents.adapters.data_model.templates.skg_templates.table_templates import serialize_df

class TableDataModel(DataModel):
    """A data model for table."""

    db_view: DataModel = None

    def set_db_view(self, db_data_model: DataModel) -> None:
        self.db_view = db_data_model

    def get_llm_side_data(self, serialize_method: str = "tsv", num_visible_rows: int = 3) -> Any:
        # Show the first few rows for observation.
        table_data = self.raw_data
        table_name = self.raw_data_name
        table_path = self.raw_data_path
        formatted_table = serialize_df(table_data, table_name, table_path, serialize_method, num_visible_rows)
        return formatted_table

    def get_human_side_data(self, mode: str = "HEAD") -> Any:
        # We support different mode for the front-end display.
        # For `HEAD` mode, we show the first few rows for observation.
        if mode == "HEAD":
            return self.raw_data.head()
        elif mode == "FULL":
            return self.raw_data
        else:
            raise ValueError(f"Unsupported mode: {mode}")

    @staticmethod
    def to_react_table(table: DataFrame) -> str:
        columns = list(map(lambda item: {"accessorKey": item, "header": item}, table.columns.tolist()))
        # FIXME: NaN may not be handled here.
        data = table.fillna("").to_dict(orient="records")
        table = json.dumps({"columns": columns, "data": data})
        return table


from __future__ import annotations

import json
from typing import Any

from pandas import DataFrame

from real_agents.adapters.data_model.base import DataModel
from real_agents.adapters.data_model.templates.skg_templates.table_templates import serialize_df

class TableDataModel(DataModel):
    """A data model for table."""

    db_view: DataModel = None

    def set_db_view(self, db_data_model: DataModel) -> None:
        self.db_view = db_data_model

    def get_llm_side_data(self, serialize_method: str = "tsv", num_visible_rows: int = 3) -> Any:
        # Show the first few rows for observation.
        table_data = self.raw_data
        table_name = self.raw_data_name
        table_path = self.raw_data_path
        formatted_table = serialize_df(table_data, table_name, table_path, serialize_method, num_visible_rows)
        return formatted_table

    def get_human_side_data(self, mode: str = "HEAD") -> Any:
        # We support different mode for the front-end display.
        # For `HEAD` mode, we show the first few rows for observation.
        if mode == "HEAD":
            return self.raw_data.head()
        elif mode == "FULL":
            return self.raw_data
        else:
            raise ValueError(f"Unsupported mode: {mode}")

    @staticmethod


from real_agents.adapters.data_model.base import DataModel
from real_agents.adapters.data_model.templates.skg_templates.table_templates import serialize_df

class TableDataModel(DataModel):
    """A data model for table."""

    db_view: DataModel = None

    def set_db_view(self, db_data_model: DataModel) -> None:
        self.db_view = db_data_model

    def get_llm_side_data(self, serialize_method: str = "tsv", num_visible_rows: int = 3) -> Any:
        # Show the first few rows for observation.
        table_data = self.raw_data
        table_name = self.raw_data_name
        table_path = self.raw_data_path
        formatted_table = serialize_df(table_data, table_name, table_path, serialize_method, num_visible_rows)
        return formatted_table

    def get_human_side_data(self, mode: str = "HEAD") -> Any:
        # We support different mode for the front-end display.
        # For `HEAD` mode, we show the first few rows for observation.
        if mode == "HEAD":
            return self.raw_data.head()
        elif mode == "FULL":
            return self.raw_data
        else:
            raise ValueError(f"Unsupported mode: {mode}")

    @staticmethod
    def to_react_table(table: DataFrame) -> str:
        columns = list(map(lambda item: {"accessorKey": item, "header": item}, table.columns.tolist()))
        # FIXME: NaN may not be handled here.
        data = table.fillna("").to_dict(orient="records")
        table = json.dumps({"columns": columns, "data": data})
        return table


    def get_llm_side_data(self, serialize_method: str = "tsv", num_visible_rows: int = 3) -> Any:
        # Show the first few rows for observation.
        table_data = self.raw_data
        table_name = self.raw_data_name
        table_path = self.raw_data_path
        formatted_table = serialize_df(table_data, table_name, table_path, serialize_method, num_visible_rows)
        return formatted_table

    def get_human_side_data(self, mode: str = "HEAD") -> Any:
        # We support different mode for the front-end display.
        # For `HEAD` mode, we show the first few rows for observation.
        if mode == "HEAD":
            return self.raw_data.head()
        elif mode == "FULL":
            return self.raw_data
        else:
            raise ValueError(f"Unsupported mode: {mode}")

    @staticmethod
    def to_react_table(table: DataFrame) -> str:
        columns = list(map(lambda item: {"accessorKey": item, "header": item}, table.columns.tolist()))
        # FIXME: NaN may not be handled here.
        data = table.fillna("").to_dict(orient="records")
        table = json.dumps({"columns": columns, "data": data})
        return table


import json
from copy import deepcopy
from typing import Dict, List

from real_agents.adapters.data_model.base import DataModel

class JsonDataModel(DataModel):
    """A data model for json, general purpose."""

    filter_keys: List[str] = []

    def get_llm_side_data(self, json_format: str = "json") -> str:
        if json_format == "json":
            assert isinstance(self.raw_data, Dict)
            llm_side_data = deepcopy(self.raw_data)
            for key, value in self.raw_data.items():
                if key in self.filter_keys:
                    llm_side_data[key] = "..."
                    continue

                if isinstance(value, DataModel):
                    llm_side_data[key] = value.get_llm_side_data()
                else:
                    llm_side_data[key] = str(value)

            return json.dumps(llm_side_data, indent=4)
        else:
            raise NotImplementedError


from __future__ import annotations

from typing import Any, Dict

import pandas as pd

from real_agents.adapters.data_model.base import DataModel
from real_agents.adapters.data_model.templates.skg_templates.database_templates import serialize_db
from real_agents.adapters.data_model.templates.skg_templates.table_templates import serialize_df
import json

class KaggleDataModel(DataModel):
    """A data model for KaggleDataModel.
    We only support the csv and sqlite format for now.
    raw_data is a Dict[str, TableDataModel]
    raw_data_path is List[str]
    raw_data_name is Dict[str, str]
    """

    def get_llm_side_data(self, serialize_method: str = "tsv", num_visible_rows: int = 3) -> Any:
        formatted_tables = []
        for _raw_data_path in self.raw_data_path:
            table_data = self.raw_data[_raw_data_path]
            table_name = self.raw_data_name[_raw_data_path]
            table_path = _raw_data_path
            formatted_table = serialize_df(table_data, table_name, table_path, serialize_method, num_visible_rows)
            formatted_tables.append(formatted_table)
        return "\n".join(formatted_tables)

    @staticmethod
    def to_react_table(table: pd.DataFrame) -> str:
        columns = list(map(lambda item: {"accessorKey": item, "header": item}, table.columns.tolist()))
        # FIXME: NaN may not be handled here.
        data = table.fillna("").to_dict(orient="records")
        table = json.dumps({"columns": columns, "data": data})
        return table

    def get_human_side_data(self) -> Any:
        # In the frontend, we show the first few rows of each table
        react_tables = {}
        for table_path in self.raw_data_path:
            table_name = self.raw_data_name[table_path]
            table = self.raw_data[table_path]
            react_tables[table_name] = self.to_react_table(table)
        return json.dumps(react_tables)


class KaggleDataModel(DataModel):
    """A data model for KaggleDataModel.
    We only support the csv and sqlite format for now.
    raw_data is a Dict[str, TableDataModel]
    raw_data_path is List[str]
    raw_data_name is Dict[str, str]
    """

    def get_llm_side_data(self, serialize_method: str = "tsv", num_visible_rows: int = 3) -> Any:
        formatted_tables = []
        for _raw_data_path in self.raw_data_path:
            table_data = self.raw_data[_raw_data_path]
            table_name = self.raw_data_name[_raw_data_path]
            table_path = _raw_data_path
            formatted_table = serialize_df(table_data, table_name, table_path, serialize_method, num_visible_rows)
            formatted_tables.append(formatted_table)
        return "\n".join(formatted_tables)

    @staticmethod
    def to_react_table(table: pd.DataFrame) -> str:
        columns = list(map(lambda item: {"accessorKey": item, "header": item}, table.columns.tolist()))
        # FIXME: NaN may not be handled here.
        data = table.fillna("").to_dict(orient="records")
        table = json.dumps({"columns": columns, "data": data})
        return table

    def get_human_side_data(self) -> Any:
        # In the frontend, we show the first few rows of each table
        react_tables = {}
        for table_path in self.raw_data_path:
            table_name = self.raw_data_name[table_path]
            table = self.raw_data[table_path]
            react_tables[table_name] = self.to_react_table(table)
        return json.dumps(react_tables)


    """

    def get_llm_side_data(self, serialize_method: str = "tsv", num_visible_rows: int = 3) -> Any:
        formatted_tables = []
        for _raw_data_path in self.raw_data_path:
            table_data = self.raw_data[_raw_data_path]
            table_name = self.raw_data_name[_raw_data_path]
            table_path = _raw_data_path
            formatted_table = serialize_df(table_data, table_name, table_path, serialize_method, num_visible_rows)
            formatted_tables.append(formatted_table)
        return "\n".join(formatted_tables)

    @staticmethod
    def to_react_table(table: pd.DataFrame) -> str:
        columns = list(map(lambda item: {"accessorKey": item, "header": item}, table.columns.tolist()))
        # FIXME: NaN may not be handled here.
        data = table.fillna("").to_dict(orient="records")
        table = json.dumps({"columns": columns, "data": data})
        return table

    def get_human_side_data(self) -> Any:
        # In the frontend, we show the first few rows of each table
        react_tables = {}
        for table_path in self.raw_data_path:
            table_name = self.raw_data_name[table_path]
            table = self.raw_data[table_path]
            react_tables[table_name] = self.to_react_table(table)
        return json.dumps(react_tables)


from typing import Any

from real_agents.adapters.data_model.base import DataModel

class ImageDataModel(DataModel):
    """A data model for image."""

    simple_filename = ""

    def get_raw_data(self) -> Any:
        return self.raw_data

    def get_llm_side_data(self) -> Any:
        if self.simple_filename == "":
            import os

            self.simple_filename = os.path.basename(self.raw_data_path)
        string = "image: " + self.simple_filename
        return string

    def get_human_side_data(self) -> Any:
        return self.raw_data["base64_string"]


from real_agents.adapters.data_model.base import DataModel

class ImageDataModel(DataModel):
    """A data model for image."""

    simple_filename = ""

    def get_raw_data(self) -> Any:
        return self.raw_data

    def get_llm_side_data(self) -> Any:
        if self.simple_filename == "":
            import os

            self.simple_filename = os.path.basename(self.raw_data_path)
        string = "image: " + self.simple_filename
        return string

    def get_human_side_data(self) -> Any:
        return self.raw_data["base64_string"]


import os
from typing import Any, Dict
import importlib.util
import tiktoken

from real_agents.adapters.data_model.plugin.base import APIYamlModel
from real_agents.adapters.data_model.utils import indent_multiline_string

def import_function_from_file(filepath, function_name):
    spec = importlib.util.spec_from_file_location("module.name", filepath)
    module = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(module)

    function = getattr(module, function_name)

    return function

def process_one_param(param_dict: Dict[str, Any]) -> str:
    name = param_dict.get("name", None)
    description = param_dict.get("description", None)
    required = param_dict.get("required", None)

    schema = param_dict.get("schema", {})
    type = schema.get("type", "UnknownType")
    value_choices = schema.get("enum", [])

    ret = (
        f"`{name}` ({type}, {'required' if required else 'optional'}): {description}."
        f"{'Examples:' + ','.join([str(_) for _ in value_choices]) if len(value_choices) > 0 else ''}"
    )
    return ret

def process_one_property(name: str, value_dict: Dict[str, Any]) -> str:
    description = value_dict.get("description", None)
    required = value_dict.get("required", None)

    type = value_dict.get("type", "UnknownType")
    value_choices = value_dict.get("enum", [])

    ret = (
        f"`{name}` ({type}, {'required' if required else 'optional'}): {description}."
        f"{'Examples:' + ','.join(value_choices) if len(value_choices) > 0 else ''}"
    )
    return ret

class SpecModel:
    def __init__(self, yaml_path: str, model_name: str = "gpt-4") -> None:
        # fixme: Must move out the logic of yaml path
        self.yaml_path = yaml_path
        self.full_spec = APIYamlModel.from_yaml(yaml_path).to_json()
        self.paths = self.full_spec["paths"]

        # Process the description
        enc = tiktoken.encoding_for_model(model_name)
        if "description" in self.full_spec["info"]:
            if len(self.full_spec["info"]["description"]) > 200:
                self.full_spec["info"]["description"] = enc.decode(
                    enc.encode(self.full_spec["info"]["description"])[:200]
                )

    def load_personnel_info(self, api_key: str):
        # Get the dir of the yaml file
        yaml_dir = os.path.dirname(self.yaml_path)
        personnel_load_dir = os.path.join(yaml_dir, "personnel.py")

        if not os.path.exists(personnel_load_dir):
            return {}, {}

        # Reload openapi.yaml
        reload_openapi = import_function_from_file(personnel_load_dir, "reload_openapi")
        resolved_json, new_paths_json = reload_openapi(api_key, self.full_spec)
        self.full_spec = resolved_json
        self.full_spec["info"] = resolved_json["info"]
        self.paths = resolved_json["paths"]

        # Reload the endpoints functions
        reload_endpoints = import_function_from_file(personnel_load_dir, "reload_endpoints")
        new_endpoint2caller = reload_endpoints(new_paths_json)

        # Reload the endpoints datamodels
        # todo: Add reload datamodels function
        new_endpoints2output_model = {k: lambda x: x for k in new_paths_json}

        return new_endpoint2caller, new_endpoints2output_model

    def prepare_spec(self, include_params: bool = True) -> str:
        path_names = list(self.paths.keys())
        ret = self.prepare_spec_for_one_path(path_names[0], include_params=include_params)

        if len(path_names) > 1:
            ret += "\n"

        for path_name in path_names[1:]:
            ret += (
                self.prepare_spec_for_one_path(path_name, include_api_info=False, include_params=include_params) + "\n"
            )

        return ret

    def list_endpoints(self) -> str:
        ret = ""
        for ep, ep_spec in self.paths.items():
            assert len(ep_spec) == 1, "Support two request methods!"
            request_method = list(ep_spec.keys())[0]
            func_spec = ep_spec[request_method]
            desc = func_spec.get("summary", None)
            ret += f"`{ep}`: {desc}\n"
        return ret.strip()

    def prepare_spec_for_one_path(
        self,
        path_name: str,
        include_api_info: bool = True,
        include_params: bool = True,
    ) -> str:
        func_dict = self.paths[path_name]
        if "servers" in func_dict:
            del func_dict["servers"]

        rets = []
        for request_method in list(func_dict.keys()):
            candidate_inputs_str = ""
            func_spec = func_dict[request_method]

            # Only GET and DELETE are processed, others are not properly processed
            if request_method.lower() not in ["get", "post", "put", "patch", "delete"]:
                raise ValueError("Unknown request method")

            # TODO: not sure how to arrange input when post method has "parameters"
            func_summary = func_spec.get("summary", None)
            func_description = func_spec.get("description", None)
            candidate_inputs = func_spec.get("parameters", [])
            candidate_inputs_str += "\n".join(process_one_param(p) for p in candidate_inputs)

            if request_method.lower() == "post" and "requestBody" in func_spec:
                request_body = func_spec["requestBody"]
                assert "content" in request_body, "Must have content in requestBody"
                content_dict = request_body["content"]
                assert len(content_dict) == 1, "Support one content type"
                content_type = list(content_dict.keys())[0]
                content = content_dict[content_type]
                assert "schema" in content, "Must have schema in requestBody"
                if "properties" in content["schema"]:
                    properties = content["schema"]["properties"]
                    candidate_inputs_str += "\n".join(process_one_property(n, vd) for n, vd in properties.items())

            ret = ""
            if include_api_info:
                ret += f"""Name: {self.full_spec["info"]["title"]}\n{'Description: ' + self.full_spec["info"]['description'] if
                "description" in self.full_spec["info"] else ""}\n"""

            ret += f"""\tSummary: {func_summary}\n"""
            ret += f"""\tDescription: {func_description}\n"""
            candidate_inputs_str = "None" if len(candidate_inputs_str) == 0 else candidate_inputs_str
            ret += (
                f"""\tInput: \n{indent_multiline_string(candidate_inputs_str, indent=2)}\n""" if include_params else ""
            )
            rets.append(ret)

        return f"""Endpoint: {path_name}\n""" + "\n".join(rets)