                    oldest_kernel_id = existing_kernel_list[0]
                    response = requests.post(
                        f"{self.base_url}/kernel/stop", json={"username": user_id, "kid": oldest_kernel_id}
                    ).json()

                    logger.bind(user_id=user_id, chat_id=chat_id, msg_head="kill oldest kernel").trace(response)

                    response = requests.post(f"{self.base_url}/kernel/create", json={"username": user_id}).json()
                cur_kid = response["id"]

                logger.bind(user_id=user_id, chat_id=chat_id, msg_head="create kernel id").trace(cur_kid)

                self.jupyter_kernel_pool.set_pool_info_with_id(
                    user_id, chat_id, {"kid": cur_kid, "ktime": time.time()}
                )

        logger.bind(user_id=user_id, chat_id=chat_id, msg_head="current kernel id").trace(cur_kid)

        return cur_kid

    def run_program_docker(
        self,
        program: str,
        kernel_id: Optional[str] = None,
        user_id: Optional[str] = "u" * 24,
        chat_id: Optional[str] = "c" * 24,
    ):
        """Run python program on the docker container(jupyter client)."""
        is_safe, ast_failed, danger_pcks = check_danger_code(program)
        if not is_safe:
            return {
                "success": False,
                "error_message": f"{ERROR_PREFIX}Unsafe Code Detected {str(danger_pcks)}, Execution Denied!",
            }
        elif ast_failed != False:
            return {
                "success": False,
                "error_message": f"{ERROR_PREFIX}Error Code Parsing, please check code grammar!\n{ast_failed}",
            }

        try:
            # Run code using remote docker jupyter kernel
            # Get the running id(i.e., a permission to run program) from redis running queue
            #  otherwise wait until a running_id is available
            p = self.r.pubsub()
            p.subscribe(RUNNING_EVENT)
            self.r.publish(SUBMIT_EVENT, chat_id)
            for message in p.listen():
                # the initial message for each channel is a message with an integer
                if isinstance(message["data"], int):
                    continue
                running_id = message["data"]
                if running_id == chat_id:
                    break
                time.sleep(1)
            # Get kernel id(i.e., the real jupyter kernel to run the program) to execute program
            cur_kid = self._apply_for_kernel(kernel_id, user_id, chat_id)
            # Execute program
            response = requests.post(
                f"{self.base_url}/kernel/exec", json={"username": user_id, "code": program, "kid": cur_kid}
            ).json()
            # Notify Redis that a job has been completed
            self.r.publish(COMPLETE_EVENT, chat_id)

            # Parse jupyter kernel output
            result, stdout, stderr, outputs, displays, error_message = None, "", "", None, [], None
            if response["status"] == "ok":
                output = response.get("output", None)
                if output is not None:
                    for output_dict in output:
                        if output_dict["type"] == "stream":
                            content = output_dict.get("content", None)
                            if content is not None:
                                if content["name"] == "stdout":
                                    stdout = content["text"]
                                if content["name"] == "stderr":
                                    stderr = content["text"]
                        elif output_dict["type"] == "execute_result":
                            content = output_dict.get("content", None)
                            if content is not None:
                                data = content.get("data", None)
                                if data is not None:
                                    if "text/plain" in data and "text/html" in data:
                                        try:
                                            # Try to recover a dataframe
                                            df = pd.read_csv(StringIO(data["text/plain"]))
                                            result = df
                                        except Exception as e:
                                            pass
                                    elif "text/plain" in data:
                                        result = data["text/plain"]
                                    else:
                                        # TODO: If not match any of the above, return the first value
                                        result = list(data.values())[0]
                        elif output_dict["type"] == "display_data":
                            content = output_dict.get("content", None)
                            if content is not None:
                                data = content.get("data", None)
                                metadata = content.get("metadata", None)
                                if data is not None and metadata is not None:
                                    if "image/png" in data:
                                        displays.append(DisplayData.from_tuple((data, metadata)))
                            if displays:
                                outputs = displays
            else:
                return {
                    "success": False,
                    "error_message": f"{ERROR_PREFIX}Error status returned by kernel",
                }
            # Check success status and return
            shell_msg = response["shell"]
            if shell_msg["status"] == "ok":
                return {
                    "success": True,
                    "result": result,
                    "stdout": stdout,
                    "stderr": stderr,
                    "outputs": outputs,
                }
            elif shell_msg["status"] == "error":
                error_message = f"{shell_msg['ename']}: {shell_msg['evalue']}"
                return {"success": False, "error_message": f"{ERROR_PREFIX}{error_message}", "outputs": outputs}
        except Exception as e:
            logger.bind(user_id=user_id, chat_id=chat_id, msg_head="Python evaluator running error").trace(e)
            import traceback

            traceback.print_exc()

            try:
                # Notify Redis that a job has been completed
                self.r.publish(COMPLETE_EVENT, chat_id)
            except:
                pass

            return {
                "success": False,
                "error_message": f"{ERROR_PREFIX}{str(e)}",
            }

    def run(
        self,
        program: str,
        environment: Optional[Any] = None,
        kernel_id: Optional[str] = None,
        user_id: Optional[str] = "u" * 24,
        chat_id: Optional[str] = "c" * 24,
    ) -> Any:
        """run generated code in certain environment"""

        lines_code = self.parse_command(program)
        program = "\n".join(lines_code)
        program = "%matplotlib inline\n" + program  # magic command to display matplotlib plots

        logger.bind(user_id=user_id, chat_id=chat_id, msg_head="Code execution mode").trace(self.code_execution_mode)

        if self.code_execution_mode == "local":
            return self.run_program_local(program, user_id)
        elif self.code_execution_mode == "docker":


run

            elif shell_msg["status"] == "error":
                error_message = f"{shell_msg['ename']}: {shell_msg['evalue']}"
                return {"success": False, "error_message": f"{ERROR_PREFIX}{error_message}", "outputs": outputs}
        except Exception as e:
            logger.bind(user_id=user_id, chat_id=chat_id, msg_head="Python evaluator running error").trace(e)
            import traceback

            traceback.print_exc()

            try:
                # Notify Redis that a job has been completed
                self.r.publish(COMPLETE_EVENT, chat_id)
            except:
                pass

            return {
                "success": False,
                "error_message": f"{ERROR_PREFIX}{str(e)}",
            }

    def run(
        self,
        program: str,
        environment: Optional[Any] = None,
        kernel_id: Optional[str] = None,
        user_id: Optional[str] = "u" * 24,
        chat_id: Optional[str] = "c" * 24,
    ) -> Any:
        """run generated code in certain environment"""

        lines_code = self.parse_command(program)
        program = "\n".join(lines_code)
        program = "%matplotlib inline\n" + program  # magic command to display matplotlib plots

        logger.bind(user_id=user_id, chat_id=chat_id, msg_head="Code execution mode").trace(self.code_execution_mode)

        if self.code_execution_mode == "local":
            return self.run_program_local(program, user_id)
        elif self.code_execution_mode == "docker":
            return self.run_program_docker(program, kernel_id, user_id, chat_id)
        else:
            raise ValueError("Invalid code execution mode")


import traceback
from typing import Any, Dict, List

from pydantic import root_validator

from real_agents.adapters.schema import SQLDatabase

class SQLEvaluator:
    """
    Util class for SQL code evaluation.
    """

    name = "SQL Evaluator"
    ERROR_PREFIX = "[ERROR]: "

    @root_validator(pre=True)
    def validate(cls, values: Dict) -> Any:
        """validate requirements for evaluation"""
        try:
            import sqlite3  # noqa F401 E402

            import sqlalchemy  # noqa F401 E402
        except ImportError:
            raise ValueError("This tool relies on sqlite3 and sqlalchemy, use `pip` to install these packages")
        return values

    @staticmethod
    def parse_command(program: str) -> List[str]:
        """patchify the code"""
        program_lines = program.strip().split("\n")
        return program_lines

    def run(self, program: str, environment: SQLDatabase) -> Any:
        """run generated code in certain environment"""
        try:
            output = environment.run(program)
            return {
                "success": True,
                "result": output,
            }
        except Exception as e:
            traceback.print_exc()
            error_message = str(e)
            return {"success": False, "error_message": f"{self.ERROR_PREFIX}{error_message}"}


class SQLEvaluator:
    """
    Util class for SQL code evaluation.
    """

    name = "SQL Evaluator"
    ERROR_PREFIX = "[ERROR]: "

    @root_validator(pre=True)
    def validate(cls, values: Dict) -> Any:
        """validate requirements for evaluation"""
        try:
            import sqlite3  # noqa F401 E402

            import sqlalchemy  # noqa F401 E402
        except ImportError:
            raise ValueError("This tool relies on sqlite3 and sqlalchemy, use `pip` to install these packages")
        return values

    @staticmethod
    def parse_command(program: str) -> List[str]:
        """patchify the code"""
        program_lines = program.strip().split("\n")
        return program_lines

    def run(self, program: str, environment: SQLDatabase) -> Any:
        """run generated code in certain environment"""
        try:
            output = environment.run(program)
            return {
                "success": True,
                "result": output,
            }
        except Exception as e:
            traceback.print_exc()
            error_message = str(e)
            return {"success": False, "error_message": f"{self.ERROR_PREFIX}{error_message}"}


    name = "SQL Evaluator"
    ERROR_PREFIX = "[ERROR]: "

    @root_validator(pre=True)
    def validate(cls, values: Dict) -> Any:
        """validate requirements for evaluation"""
        try:
            import sqlite3  # noqa F401 E402

            import sqlalchemy  # noqa F401 E402
        except ImportError:
            raise ValueError("This tool relies on sqlite3 and sqlalchemy, use `pip` to install these packages")
        return values

    @staticmethod
    def parse_command(program: str) -> List[str]:
        """patchify the code"""
        program_lines = program.strip().split("\n")
        return program_lines

    def run(self, program: str, environment: SQLDatabase) -> Any:
        """run generated code in certain environment"""
        try:
            output = environment.run(program)
            return {
                "success": True,
                "result": output,
            }
        except Exception as e:
            traceback.print_exc()
            error_message = str(e)
            return {"success": False, "error_message": f"{self.ERROR_PREFIX}{error_message}"}


import json
import os
import re
import shutil
import uuid
from typing import Any, Dict, List, Tuple
import requests
from bs4 import BeautifulSoup
from loguru import logger
from kaggle.api.kaggle_api_extended import KaggleApi

from langchain.base_language import BaseLanguageModel
from langchain import PromptTemplate

from real_agents.adapters.llm import LLMChain

class KaggleDataLoadingExecutor:
    KAGGLE_TEMPLATE = """

Determine whether the user input aims to (1) connect to a specific kaggle dataset that the user mentions its kaggle path
(2) search for relevant kaggle datasets given the information the user provides.

You need to output the action wrapped in <action></action>, the action space is ['connect', 'search']. You also need
to output the keywords wrapped in <keywords></keywords>. For 'search', the keywords MUST be ONE search term/word to be
searched by kaggle api. Note keywords CAN'T be too specific or contain trivial word(e.g., dataset), make sure there are various search results. For
'connect', the keywords are the kaggle dataset path.

Input: {input}

Begin."
"""

    def run(
        self,
        user_intent: str,
        llm: BaseLanguageModel,
        search_top_k: int = 4,
    ) -> Dict[str, Any]:
        logger.bind(msg_head="KaggleDataLoader inputs").trace(user_intent)
        kaggle_template = PromptTemplate(
            input_variables=["input"],
            template=self.KAGGLE_TEMPLATE,
        )
        method = LLMChain(llm=llm, prompt=kaggle_template)
        result = method.run({"input": user_intent})
        logger.bind(msg_head="LLM result").trace(result)
        kaggle_action, keywords = self._parse_output(result)
        logger.bind(msg_head="Kaggle action").trace(kaggle_action)
        logger.bind(msg_head="Kaggle keywords").trace(keywords)
        """Use export to manage the Kaggle API key for now."""
        api = KaggleApi()
        api.authenticate()
        if kaggle_action == "connect":
            kaggle_output_info = keywords
        elif kaggle_action == "search":
            kaggle_output_info = self._search_kaggle(api, keywords, search_top_k)
        else:
            # Regard the rest as "search" action now
            kaggle_action = "search"
            kaggle_output_info = self._search_kaggle(api, keywords, search_top_k)
        return {"kaggle_action": kaggle_action, "kaggle_output_info": kaggle_output_info}

    def _search_kaggle(self, api: KaggleApi, keywords: str, search_top_k: int) -> List[Dict]:
        """Search kaggle datasets given the keywords."""
        # Search for datasets
        datasets = []
        for page in range(1, 10):
            try:
                searched_datasets = api.dataset_list(search=keywords, page=page, max_size=20000, file_type="csv")

                logger.bind(msg_head="Kaggle search result").trace(searched_datasets)

                datasets.extend(searched_datasets)
                if len(datasets) >= search_top_k:
                    datasets = datasets[:search_top_k]
                    break
                if len(searched_datasets) < 20:
                    # Default page_size is 20, less than 20 means no more datasets can be searched
                    break
            except Exception:
                break

        # Get url, cover image and some meta data for each dataset
        if len(datasets) == 0:
            # No datasets found
            datasets = api.dataset_list(max_size=20000, page=1, file_type="csv")[:search_top_k]

        output_info = self._get_dataset_meta_info(api, datasets)
        return output_info

    def _get_dataset_meta_info(self, api: KaggleApi, datasets: List) -> List[Dict]:
        """Get dataset key meta-data to be shown to the user."""
        output_info = []
        for dataset in datasets:
            dataset_hash_id = str(uuid.uuid4())
            dataset_tmp_dir = os.path.join(".kaggle_meta/", dataset_hash_id)
            os.makedirs(dataset_tmp_dir, exist_ok=True)
            api.dataset_metadata(dataset.ref, path=dataset_tmp_dir)
            with open(os.path.join(dataset_tmp_dir, "dataset-metadata.json")) as f:
                dataset_metadata = json.load(f)
            shutil.rmtree(os.path.join(".kaggle_meta/", dataset_hash_id))
            dataset_url = "https://www.kaggle.com/datasets/" + dataset.ref
            # Crawling the dataset page to get the dataset image
            dataset_cover_image_url = self._crawl_dataset_cover_image(dataset_url)

            logger.bind(msg_head="Dataset cover image url").trace(dataset_cover_image_url)

            output_metadata = {
                "id": dataset_metadata["id"],
                "id_no": dataset_metadata["id_no"],
                "title": dataset_metadata["title"],
                "subtitle": dataset_metadata["subtitle"],
                "total_views": dataset_metadata["totalViews"],
                "total_votes": dataset_metadata["totalVotes"],
                "total_downloads": dataset_metadata["totalDownloads"],
                "url": dataset_url,
                "cover_image_url": dataset_cover_image_url,
            }
            output_info.append(output_metadata)
        return output_info

    def _crawl_dataset_cover_image(
        self, url: str, default_image_path="https://images.datacamp.com/image/upload/v1647430873/kaggle_logo_icon_168474_4eb653edb6.png"
    ) -> str:
        """Crawl the kaggle dataset cover image from the dataset url."""
        # Get the HTML content of the webpage
        response = requests.get(url)

        # Parse the HTML with BeautifulSoup
        soup = BeautifulSoup(response.text, "html.parser")

        # Find the image element
        try:
            kaggle_component_element = soup.find("script", {"class": "kaggle-component"})
            match = re.search(r'"coverImageUrl":\s*"([^"]*)"', kaggle_component_element.string)
            image_url = match.group(1)
        except Exception:
            import traceback

            traceback.print_exc()
            image_url = default_image_path

        return image_url

    def _parse_output(self, content: str) -> Tuple[str, str]:
        """Parse the output of the LLM to get the kaggle action and keywords."""
        from bs4 import BeautifulSoup

        # Using 'html.parser' to parse the content
        soup = BeautifulSoup(content, "html.parser")
        # Parsing the tag and summary contents
        try:
            action = soup.find("action").text
        except Exception:
            action = ""

        try:
            keywords = soup.find("keywords").text
        except Exception:
            keywords = ""

        return action, keywords


from real_agents.adapters.llm import LLMChain

class KaggleDataLoadingExecutor:
    KAGGLE_TEMPLATE = """

Determine whether the user input aims to (1) connect to a specific kaggle dataset that the user mentions its kaggle path
(2) search for relevant kaggle datasets given the information the user provides.

You need to output the action wrapped in <action></action>, the action space is ['connect', 'search']. You also need
to output the keywords wrapped in <keywords></keywords>. For 'search', the keywords MUST be ONE search term/word to be
searched by kaggle api. Note keywords CAN'T be too specific or contain trivial word(e.g., dataset), make sure there are various search results. For
'connect', the keywords are the kaggle dataset path.

Input: {input}

Begin."
"""

    def run(
        self,
        user_intent: str,
        llm: BaseLanguageModel,
        search_top_k: int = 4,
    ) -> Dict[str, Any]:
        logger.bind(msg_head="KaggleDataLoader inputs").trace(user_intent)
        kaggle_template = PromptTemplate(
            input_variables=["input"],
            template=self.KAGGLE_TEMPLATE,
        )
        method = LLMChain(llm=llm, prompt=kaggle_template)
        result = method.run({"input": user_intent})
        logger.bind(msg_head="LLM result").trace(result)
        kaggle_action, keywords = self._parse_output(result)
        logger.bind(msg_head="Kaggle action").trace(kaggle_action)
        logger.bind(msg_head="Kaggle keywords").trace(keywords)
        """Use export to manage the Kaggle API key for now."""
        api = KaggleApi()
        api.authenticate()
        if kaggle_action == "connect":
            kaggle_output_info = keywords
        elif kaggle_action == "search":
            kaggle_output_info = self._search_kaggle(api, keywords, search_top_k)
        else:
            # Regard the rest as "search" action now
            kaggle_action = "search"
            kaggle_output_info = self._search_kaggle(api, keywords, search_top_k)
        return {"kaggle_action": kaggle_action, "kaggle_output_info": kaggle_output_info}

    def _search_kaggle(self, api: KaggleApi, keywords: str, search_top_k: int) -> List[Dict]:
        """Search kaggle datasets given the keywords."""
        # Search for datasets
        datasets = []
        for page in range(1, 10):
            try:
                searched_datasets = api.dataset_list(search=keywords, page=page, max_size=20000, file_type="csv")

                logger.bind(msg_head="Kaggle search result").trace(searched_datasets)

                datasets.extend(searched_datasets)
                if len(datasets) >= search_top_k:
                    datasets = datasets[:search_top_k]
                    break
                if len(searched_datasets) < 20:
                    # Default page_size is 20, less than 20 means no more datasets can be searched
                    break
            except Exception:
                break


        )
        method = LLMChain(llm=llm, prompt=kaggle_template)
        result = method.run({"input": user_intent})
        logger.bind(msg_head="LLM result").trace(result)
        kaggle_action, keywords = self._parse_output(result)
        logger.bind(msg_head="Kaggle action").trace(kaggle_action)
        logger.bind(msg_head="Kaggle keywords").trace(keywords)
        """Use export to manage the Kaggle API key for now."""
        api = KaggleApi()
        api.authenticate()
        if kaggle_action == "connect":
            kaggle_output_info = keywords
        elif kaggle_action == "search":
            kaggle_output_info = self._search_kaggle(api, keywords, search_top_k)
        else:
            # Regard the rest as "search" action now
            kaggle_action = "search"
            kaggle_output_info = self._search_kaggle(api, keywords, search_top_k)
        return {"kaggle_action": kaggle_action, "kaggle_output_info": kaggle_output_info}

    def _search_kaggle(self, api: KaggleApi, keywords: str, search_top_k: int) -> List[Dict]:
        """Search kaggle datasets given the keywords."""
        # Search for datasets
        datasets = []
        for page in range(1, 10):
            try:
                searched_datasets = api.dataset_list(search=keywords, page=page, max_size=20000, file_type="csv")

                logger.bind(msg_head="Kaggle search result").trace(searched_datasets)

                datasets.extend(searched_datasets)
                if len(datasets) >= search_top_k:
                    datasets = datasets[:search_top_k]
                    break
                if len(searched_datasets) < 20:
                    # Default page_size is 20, less than 20 means no more datasets can be searched
                    break
            except Exception:
                break

        # Get url, cover image and some meta data for each dataset
        if len(datasets) == 0:
            # No datasets found
            datasets = api.dataset_list(max_size=20000, page=1, file_type="csv")[:search_top_k]

        output_info = self._get_dataset_meta_info(api, datasets)
        return output_info

    def _get_dataset_meta_info(self, api: KaggleApi, datasets: List) -> List[Dict]:
        """Get dataset key meta-data to be shown to the user."""
        output_info = []
        for dataset in datasets:
            dataset_hash_id = str(uuid.uuid4())
            dataset_tmp_dir = os.path.join(".kaggle_meta/", dataset_hash_id)
            os.makedirs(dataset_tmp_dir, exist_ok=True)
            api.dataset_metadata(dataset.ref, path=dataset_tmp_dir)
            with open(os.path.join(dataset_tmp_dir, "dataset-metadata.json")) as f:
                dataset_metadata = json.load(f)
            shutil.rmtree(os.path.join(".kaggle_meta/", dataset_hash_id))
            dataset_url = "https://www.kaggle.com/datasets/" + dataset.ref
            # Crawling the dataset page to get the dataset image
            dataset_cover_image_url = self._crawl_dataset_cover_image(dataset_url)

            logger.bind(msg_head="Dataset cover image url").trace(dataset_cover_image_url)

            output_metadata = {
                "id": dataset_metadata["id"],


                logger.bind(msg_head="Kaggle search result").trace(searched_datasets)

                datasets.extend(searched_datasets)
                if len(datasets) >= search_top_k:
                    datasets = datasets[:search_top_k]
                    break
                if len(searched_datasets) < 20:
                    # Default page_size is 20, less than 20 means no more datasets can be searched
                    break
            except Exception:
                break

        # Get url, cover image and some meta data for each dataset
        if len(datasets) == 0:
            # No datasets found
            datasets = api.dataset_list(max_size=20000, page=1, file_type="csv")[:search_top_k]

        output_info = self._get_dataset_meta_info(api, datasets)
        return output_info

    def _get_dataset_meta_info(self, api: KaggleApi, datasets: List) -> List[Dict]:
        """Get dataset key meta-data to be shown to the user."""
        output_info = []
        for dataset in datasets:
            dataset_hash_id = str(uuid.uuid4())
            dataset_tmp_dir = os.path.join(".kaggle_meta/", dataset_hash_id)
            os.makedirs(dataset_tmp_dir, exist_ok=True)
            api.dataset_metadata(dataset.ref, path=dataset_tmp_dir)
            with open(os.path.join(dataset_tmp_dir, "dataset-metadata.json")) as f:
                dataset_metadata = json.load(f)
            shutil.rmtree(os.path.join(".kaggle_meta/", dataset_hash_id))
            dataset_url = "https://www.kaggle.com/datasets/" + dataset.ref
            # Crawling the dataset page to get the dataset image
            dataset_cover_image_url = self._crawl_dataset_cover_image(dataset_url)

            logger.bind(msg_head="Dataset cover image url").trace(dataset_cover_image_url)

            output_metadata = {
                "id": dataset_metadata["id"],
                "id_no": dataset_metadata["id_no"],
                "title": dataset_metadata["title"],
                "subtitle": dataset_metadata["subtitle"],
                "total_views": dataset_metadata["totalViews"],
                "total_votes": dataset_metadata["totalVotes"],
                "total_downloads": dataset_metadata["totalDownloads"],
                "url": dataset_url,
                "cover_image_url": dataset_cover_image_url,
            }
            output_info.append(output_metadata)
        return output_info

    def _crawl_dataset_cover_image(
        self, url: str, default_image_path="https://images.datacamp.com/image/upload/v1647430873/kaggle_logo_icon_168474_4eb653edb6.png"
    ) -> str:
        """Crawl the kaggle dataset cover image from the dataset url."""
        # Get the HTML content of the webpage
        response = requests.get(url)

        # Parse the HTML with BeautifulSoup
        soup = BeautifulSoup(response.text, "html.parser")

        # Find the image element
        try:
            kaggle_component_element = soup.find("script", {"class": "kaggle-component"})
            match = re.search(r'"coverImageUrl":\s*"([^"]*)"', kaggle_component_element.string)
            image_url = match.group(1)
        except Exception:
            import traceback

            traceback.print_exc()


            dataset_url = "https://www.kaggle.com/datasets/" + dataset.ref
            # Crawling the dataset page to get the dataset image
            dataset_cover_image_url = self._crawl_dataset_cover_image(dataset_url)

            logger.bind(msg_head="Dataset cover image url").trace(dataset_cover_image_url)

            output_metadata = {
                "id": dataset_metadata["id"],
                "id_no": dataset_metadata["id_no"],
                "title": dataset_metadata["title"],
                "subtitle": dataset_metadata["subtitle"],
                "total_views": dataset_metadata["totalViews"],
                "total_votes": dataset_metadata["totalVotes"],
                "total_downloads": dataset_metadata["totalDownloads"],
                "url": dataset_url,
                "cover_image_url": dataset_cover_image_url,
            }
            output_info.append(output_metadata)
        return output_info

    def _crawl_dataset_cover_image(
        self, url: str, default_image_path="https://images.datacamp.com/image/upload/v1647430873/kaggle_logo_icon_168474_4eb653edb6.png"
    ) -> str:
        """Crawl the kaggle dataset cover image from the dataset url."""
        # Get the HTML content of the webpage
        response = requests.get(url)

        # Parse the HTML with BeautifulSoup
        soup = BeautifulSoup(response.text, "html.parser")

        # Find the image element
        try:
            kaggle_component_element = soup.find("script", {"class": "kaggle-component"})
            match = re.search(r'"coverImageUrl":\s*"([^"]*)"', kaggle_component_element.string)
            image_url = match.group(1)
        except Exception:
            import traceback

            traceback.print_exc()
            image_url = default_image_path

        return image_url

    def _parse_output(self, content: str) -> Tuple[str, str]:
        """Parse the output of the LLM to get the kaggle action and keywords."""
        from bs4 import BeautifulSoup

        # Using 'html.parser' to parse the content
        soup = BeautifulSoup(content, "html.parser")
        # Parsing the tag and summary contents
        try:
            action = soup.find("action").text
        except Exception:
            action = ""

        try:
            keywords = soup.find("keywords").text
        except Exception:
            keywords = ""

        return action, keywords


        """Crawl the kaggle dataset cover image from the dataset url."""
        # Get the HTML content of the webpage
        response = requests.get(url)

        # Parse the HTML with BeautifulSoup
        soup = BeautifulSoup(response.text, "html.parser")

        # Find the image element
        try:
            kaggle_component_element = soup.find("script", {"class": "kaggle-component"})
            match = re.search(r'"coverImageUrl":\s*"([^"]*)"', kaggle_component_element.string)
            image_url = match.group(1)
        except Exception:
            import traceback

            traceback.print_exc()
            image_url = default_image_path

        return image_url

    def _parse_output(self, content: str) -> Tuple[str, str]:
        """Parse the output of the LLM to get the kaggle action and keywords."""
        from bs4 import BeautifulSoup

        # Using 'html.parser' to parse the content
        soup = BeautifulSoup(content, "html.parser")
        # Parsing the tag and summary contents
        try:
            action = soup.find("action").text
        except Exception:
            action = ""

        try:
            keywords = soup.find("keywords").text
        except Exception:
            keywords = ""

        return action, keywords


from typing import Any, Dict, List, Literal, Optional, Union

from langchain.base_language import BaseLanguageModel

from real_agents.adapters.data_model import DatabaseDataModel, TableDataModel, ImageDataModel
from real_agents.adapters.memory import ReadOnlySharedStringMemory
from real_agents.adapters.schema import SQLDatabase
from real_agents.data_agent.python.base import PythonChain
from real_agents.data_agent.sql.base import SQLDatabaseChain

class CodeGenerationExecutor:
    """Code Generation Executor.

    Example:
        .. code-block:: python

                from real_agents.adapters.executors import CodeGenerationExecutor
                executor = CodeGenerationExecutor(programming_language="sql")
                executor.run(
                    user_intent="What is the name of the first employee?",
                    grounding_source=SQLDatabase.from_uri(...)
                )

    """

    def __init__(
        self,
        programming_language: Literal["sql", "python"],
        usage: Union[None, str] = None,
        example_selector: Any = None,
        memory: Optional[ReadOnlySharedStringMemory] = None,
    ) -> None:
        """Initialize the executor.

        Args:
            programming_language: Programming language to generate.
            example_selector: Example selector to select few-shot in-context exemplars.
        """
        self._programming_language = programming_language
        self._usage = usage
        self._example_selector = example_selector
        self._memory = memory

    @property
    def programming_language(self) -> str:
        """Get programming language."""
        return self._programming_language

    def run(
        self,
        user_intent: str,
        llm: BaseLanguageModel,
        grounding_source: Optional[Union[List[TableDataModel], DatabaseDataModel, ImageDataModel]] = None,
        user_id: str = None,
        chat_id: str = None,
        code_execution_mode: str = "local",
        jupyter_kernel_pool: Any = None,
        return_intermediate_steps: bool = True,
        return_direct: bool = True,
        verbose: bool = True,
    ) -> Dict[str, Any]:
        """Run the executor.

        Args:
            user_intent: User intent to execute.
            grounding_source: Grounding source to execute the program on. should be {file_name: data}
            llm: Language model to use.
            return_intermediate_steps: Whether to return the intermediate steps, e.g., the program.
            return_direct: Whether to return the result of program execution directly.
            verbose: Whether to print the logging.

        Returns:
            Result dictionary of code generation
        """

        def _concat_grounding_source() -> str:
            assert isinstance(grounding_source, list)
            table_schema = ""
            for gs in grounding_source:
                table_schema += f"{gs.get_llm_side_data()}\n"
            return table_schema

        if self._programming_language == "sql":
            db = grounding_source.raw_data
            assert isinstance(db, SQLDatabase)
            method = SQLDatabaseChain(
                llm=llm,
                database=db,
                example_selector=self._example_selector,
                memory=self._memory,
                return_direct=return_direct,
                return_intermediate_steps=return_intermediate_steps,
                verbose=verbose,
            )
            _input = {"user_intent": user_intent}
            result = method(_input)
        elif self._programming_language == "python":
            if self._usage is None:
                # General python code generation for data analysis
                method = PythonChain.from_python_prompt(
                    llm,
                    return_intermediate_steps=return_intermediate_steps,
                    verbose=True,
                    memory=self._memory,
                    user_id=user_id,
                    chat_id=chat_id,
                    code_execution_mode=code_execution_mode,
                    jupyter_kernel_pool=jupyter_kernel_pool,
                )
                # Get each source_item (table, db, files...) from the grounding_source
                _input = {"question": user_intent, "data_info": _concat_grounding_source()}
                result = method(_input)
            elif self._usage == "echarts":
                # Python code generation for echarts interactive chart
                method = PythonChain.from_echarts_prompt(
                    llm,
                    return_intermediate_steps=return_intermediate_steps,
                    verbose=True,
                    memory=self._memory,
                    user_id=user_id,
                    chat_id=chat_id,
                    code_execution_mode=code_execution_mode,
                    jupyter_kernel_pool=jupyter_kernel_pool,
                )
                _input = {"question": user_intent, "data_info": _concat_grounding_source()}
                result = method(_input)
            else:
                raise ValueError(f"Usage {self._usage} not supported yet.")
        else:
            raise ValueError(f"Programming language {self._programming_language} not supported.")
        return result


from real_agents.adapters.schema import SQLDatabase
from real_agents.data_agent.python.base import PythonChain
from real_agents.data_agent.sql.base import SQLDatabaseChain

class CodeGenerationExecutor:
    """Code Generation Executor.

    Example:
        .. code-block:: python

                from real_agents.adapters.executors import CodeGenerationExecutor
                executor = CodeGenerationExecutor(programming_language="sql")
                executor.run(
                    user_intent="What is the name of the first employee?",
                    grounding_source=SQLDatabase.from_uri(...)
                )

    """

    def __init__(
        self,
        programming_language: Literal["sql", "python"],
        usage: Union[None, str] = None,
        example_selector: Any = None,
        memory: Optional[ReadOnlySharedStringMemory] = None,
    ) -> None:
        """Initialize the executor.

        Args:
            programming_language: Programming language to generate.
            example_selector: Example selector to select few-shot in-context exemplars.
        """
        self._programming_language = programming_language
        self._usage = usage
        self._example_selector = example_selector
        self._memory = memory

    @property
    def programming_language(self) -> str:
        """Get programming language."""
        return self._programming_language

    def run(
        self,
        user_intent: str,
        llm: BaseLanguageModel,
        grounding_source: Optional[Union[List[TableDataModel], DatabaseDataModel, ImageDataModel]] = None,
        user_id: str = None,
        chat_id: str = None,
        code_execution_mode: str = "local",
        jupyter_kernel_pool: Any = None,
        return_intermediate_steps: bool = True,
        return_direct: bool = True,
        verbose: bool = True,
    ) -> Dict[str, Any]:
        """Run the executor.


    def __init__(
        self,
        programming_language: Literal["sql", "python"],
        usage: Union[None, str] = None,
        example_selector: Any = None,
        memory: Optional[ReadOnlySharedStringMemory] = None,
    ) -> None:
        """Initialize the executor.

        Args:
            programming_language: Programming language to generate.
            example_selector: Example selector to select few-shot in-context exemplars.
        """
        self._programming_language = programming_language
        self._usage = usage
        self._example_selector = example_selector
        self._memory = memory

    @property
    def programming_language(self) -> str:
        """Get programming language."""
        return self._programming_language

    def run(
        self,
        user_intent: str,
        llm: BaseLanguageModel,
        grounding_source: Optional[Union[List[TableDataModel], DatabaseDataModel, ImageDataModel]] = None,
        user_id: str = None,
        chat_id: str = None,
        code_execution_mode: str = "local",
        jupyter_kernel_pool: Any = None,
        return_intermediate_steps: bool = True,
        return_direct: bool = True,
        verbose: bool = True,
    ) -> Dict[str, Any]:
        """Run the executor.

        Args:
            user_intent: User intent to execute.
            grounding_source: Grounding source to execute the program on. should be {file_name: data}
            llm: Language model to use.


        usage: Union[None, str] = None,
        example_selector: Any = None,
        memory: Optional[ReadOnlySharedStringMemory] = None,
    ) -> None:
        """Initialize the executor.

        Args:
            programming_language: Programming language to generate.
            example_selector: Example selector to select few-shot in-context exemplars.
        """
        self._programming_language = programming_language
        self._usage = usage
        self._example_selector = example_selector
        self._memory = memory

    @property
    def programming_language(self) -> str:
        """Get programming language."""
        return self._programming_language

    def run(
        self,
        user_intent: str,
        llm: BaseLanguageModel,
        grounding_source: Optional[Union[List[TableDataModel], DatabaseDataModel, ImageDataModel]] = None,
        user_id: str = None,
        chat_id: str = None,
        code_execution_mode: str = "local",
        jupyter_kernel_pool: Any = None,
        return_intermediate_steps: bool = True,
        return_direct: bool = True,
        verbose: bool = True,
    ) -> Dict[str, Any]:
        """Run the executor.

        Args:
            user_intent: User intent to execute.
            grounding_source: Grounding source to execute the program on. should be {file_name: data}
            llm: Language model to use.
            return_intermediate_steps: Whether to return the intermediate steps, e.g., the program.
            return_direct: Whether to return the result of program execution directly.
            verbose: Whether to print the logging.

        Returns:
            Result dictionary of code generation
        """

        def _concat_grounding_source() -> str:
            assert isinstance(grounding_source, list)
            table_schema = ""
            for gs in grounding_source:
                table_schema += f"{gs.get_llm_side_data()}\n"
            return table_schema

        if self._programming_language == "sql":
            db = grounding_source.raw_data
            assert isinstance(db, SQLDatabase)
            method = SQLDatabaseChain(
                llm=llm,
                database=db,
                example_selector=self._example_selector,
                memory=self._memory,
                return_direct=return_direct,
                return_intermediate_steps=return_intermediate_steps,
                verbose=verbose,
            )
            _input = {"user_intent": user_intent}
            result = method(_input)
        elif self._programming_language == "python":
            if self._usage is None:
                # General python code generation for data analysis
                method = PythonChain.from_python_prompt(
                    llm,
                    return_intermediate_steps=return_intermediate_steps,
                    verbose=True,
                    memory=self._memory,
                    user_id=user_id,
                    chat_id=chat_id,
                    code_execution_mode=code_execution_mode,
                    jupyter_kernel_pool=jupyter_kernel_pool,
                )
                # Get each source_item (table, db, files...) from the grounding_source
                _input = {"question": user_intent, "data_info": _concat_grounding_source()}
                result = method(_input)
            elif self._usage == "echarts":
                # Python code generation for echarts interactive chart
                method = PythonChain.from_echarts_prompt(
                    llm,
                    return_intermediate_steps=return_intermediate_steps,
                    verbose=True,
                    memory=self._memory,
                    user_id=user_id,
                    chat_id=chat_id,
                    code_execution_mode=code_execution_mode,
                    jupyter_kernel_pool=jupyter_kernel_pool,
                )
                _input = {"question": user_intent, "data_info": _concat_grounding_source()}
                result = method(_input)
            else:
                raise ValueError(f"Usage {self._usage} not supported yet.")
        else:
            raise ValueError(f"Programming language {self._programming_language} not supported.")
        return result


        code_execution_mode: str = "local",
        jupyter_kernel_pool: Any = None,
        return_intermediate_steps: bool = True,
        return_direct: bool = True,
        verbose: bool = True,
    ) -> Dict[str, Any]:
        """Run the executor.

        Args:
            user_intent: User intent to execute.
            grounding_source: Grounding source to execute the program on. should be {file_name: data}
            llm: Language model to use.
            return_intermediate_steps: Whether to return the intermediate steps, e.g., the program.
            return_direct: Whether to return the result of program execution directly.
            verbose: Whether to print the logging.

        Returns:
            Result dictionary of code generation
        """

        def _concat_grounding_source() -> str:
            assert isinstance(grounding_source, list)
            table_schema = ""
            for gs in grounding_source:
                table_schema += f"{gs.get_llm_side_data()}\n"
            return table_schema

        if self._programming_language == "sql":
            db = grounding_source.raw_data
            assert isinstance(db, SQLDatabase)
            method = SQLDatabaseChain(
                llm=llm,
                database=db,
                example_selector=self._example_selector,
                memory=self._memory,
                return_direct=return_direct,
                return_intermediate_steps=return_intermediate_steps,
                verbose=verbose,
            )
            _input = {"user_intent": user_intent}
            result = method(_input)
        elif self._programming_language == "python":
            if self._usage is None:
                # General python code generation for data analysis
                method = PythonChain.from_python_prompt(
                    llm,


from typing import Any, Dict, Tuple, Union
from abc import ABC

from langchain.base_language import BaseLanguageModel
from langchain import PromptTemplate

from real_agents.adapters.callbacks.executor_streaming import ExecutorStreamingChainHandler
from real_agents.adapters.data_model import DatabaseDataModel, TableDataModel, ImageDataModel
from real_agents.adapters.llm import LLMChain

class DataSummaryExecutor(ABC):
    tool_name = "DataProfiling"

    def _intelligent_summary(self, grounding_source: ImageDataModel, num_insights: int, llm: BaseLanguageModel) -> str:
        """Use LLM to generate data summary."""
        pass

class TableSummaryExecutor(DataSummaryExecutor):
    SUMMARY_PROMPT_TEMPLATE = """
{table_info}

Provide a succinct yet meaningful summary of the table with less than 20 words, encapsulating its essence beyond just enumerating the columns. Please ensure your summary is a complete sentence and include it within  tags."
Note the table actually far more rows than shown above, so you MUST NOT make any rash conclusions based on the shown table rows or cells."
Then provide {num_insights} insightful and interesting suggestions in natural language that users can directly say to analyze the table. The suggestions should be able to be solved by python/sql."
The final results should be markdown '+' bullet point list, e.g., + The first suggestion.

Begin."
"""
    stream_handler = ExecutorStreamingChainHandler()

    def run(
        self,
        grounding_source: Union[TableDataModel, DatabaseDataModel],
        llm: BaseLanguageModel,
        use_intelligent_summary: bool = True,
        num_insights: int = 3,
    ) -> Dict[str, Any]:
        summary = ""
        if isinstance(grounding_source, TableDataModel):
            df = grounding_source.raw_data
            df_name = grounding_source.raw_data_name
            # Basic summary
            summary += f"Your table {df_name} contains {df.shape[0]} rows and {df.shape[1]} columns. "

            null_count = df.isnull().sum().sum()  # Get total number of null values
            unique_values_avg = df.nunique().mean()  # Get average number of unique values

            summary += f"On average, each column has about {unique_values_avg:.0f} unique values. "
            if null_count > 0:
                summary += f"Watch out, there are {null_count} missing values in your data. "
            else:
                summary += "Good news, no missing values in your data. "

            # Intelligent summary
            if use_intelligent_summary:
                intelligent_summary = self._intelligent_summary(
                    grounding_source,
                    num_insights=num_insights,
                    llm=llm,
                )
                table_summary, suggestions = self._parse_output(intelligent_summary)
                summary += table_summary
                summary += "\n" + "Here are some additional insights to enhance your understanding of the table."
                summary += "\n" + suggestions

            for stream_token in summary.split(" "):
                self.stream_handler.on_llm_new_token(stream_token)

        elif isinstance(grounding_source, DatabaseDataModel):
            # TODO: Convert to df or use SQL query for basic summary
            raise NotImplementedError("DatabaseDataModel is not supported yet.")
        else:
            raise ValueError(f"Unsupported grounding source type: {type(grounding_source)}")
        return summary

    def _intelligent_summary(
        self, grounding_source: Union[TableDataModel, DatabaseDataModel], num_insights: int, llm: BaseLanguageModel
    ) -> str:
        """Use LLM to generate data summary."""
        summary_prompt_template = PromptTemplate(
            input_variables=["table_info", "num_insights"],
            template=self.SUMMARY_PROMPT_TEMPLATE,
        )
        method = LLMChain(llm=llm, prompt=summary_prompt_template)
        result = method.run({"table_info": grounding_source.get_llm_side_data(), "num_insights": num_insights})
        return result

    def _parse_output(self, content: str) -> Tuple[str, str]:
        """Parse the output of the LLM to get the data summary."""
        from bs4 import BeautifulSoup

        # Using 'html.parser' to parse the content
        soup = BeautifulSoup(content, "html.parser")
        # Parsing the tag and summary contents
        try:
            table_summary = soup.find("summary").text
        except Exception:
            import traceback

            traceback.print_exc()
            table_summary = ""

        lines = content.split("\n")
        # Initialize an empty list to hold the parsed bullet points
        bullet_points = []
        # Loop through each line
        bullet_point_id = 1
        for line in lines:
            # If the line starts with '+', it is a bullet point
            if line.startswith("+"):
                # Remove the '+ ' from the start of the line and add it to the list
                bullet_points.append(f"{bullet_point_id}. " + line[1:].strip().strip('"'))
                bullet_point_id += 1
        return table_summary, "\n".join(bullet_points)

class ImageSummaryExecutor(DataSummaryExecutor):
    SUMMARY_PROMPT_TEMPLATE = """
{img_info}

Provide a succinct summary of the uploaded file with less than 20 words. Please ensure your summary is a complete sentence and include it within  tags. For image, just show its name is basically enough."
Then provide {num_insights} very simple and basic suggestions in natural language about further processing with the data. The suggestions should be able to be solved by python(e.g., grayscale, rescale, rotation, etc). The final results should be markdown '+' bullet point list, e.g., + The first suggestion."

Begin.
"""
    stream_handler = ExecutorStreamingChainHandler()

    def run(
        self,
        grounding_source: ImageDataModel,
        llm: BaseLanguageModel,
        use_intelligent_summary: bool = True,
        num_insights: int = 3,
    ) -> Dict[str, Any]:
        summary = ""
        if isinstance(grounding_source, ImageDataModel):
            # Basic summary
            raw_data = grounding_source.raw_data
            img_size, img_mode, img_format = raw_data["size"], raw_data["mode"], raw_data["format"]
            summary += f"Your image **{grounding_source.simple_filename}** is a {img_size[0]}x{img_size[1]} {img_mode} image in {img_format} format.\n"

            # Intelligent summary
            if use_intelligent_summary:
                intelligent_summary = self._intelligent_summary(
                    grounding_source,
                    num_insights=num_insights,
                    llm=llm,
                )
                _, suggestions = self._parse_output(intelligent_summary)
                summary += "\n" + "Here are some additional insights to enhance your understanding of the image"
                summary += "\n" + suggestions

            for stream_token in summary.split(" "):
                self.stream_handler.on_llm_new_token(stream_token)
        else:
            raise ValueError(f"Unsupported data summary for grounding source type: {type(grounding_source)}")
        return summary

    def _intelligent_summary(self, grounding_source: ImageDataModel, num_insights: int, llm: BaseLanguageModel) -> str:
        """Use LLM to generate data summary."""
        summary_prompt_template = PromptTemplate(
            input_variables=["img_info", "num_insights"],
            template=self.SUMMARY_PROMPT_TEMPLATE,
        )
        method = LLMChain(llm=llm, prompt=summary_prompt_template)
        result = method.run({"img_info": grounding_source.get_llm_side_data(), "num_insights": num_insights})
        return result

    def _parse_output(self, content: str) -> Tuple[str, str]:
        """Parse the output of the LLM to get the data summary."""
        from bs4 import BeautifulSoup

        # Using 'html.parser' to parse the content
        soup = BeautifulSoup(content, "html.parser")
        # Parsing the tag and summary contents
        try:
            table_summary = soup.find("summary").text
        except Exception:
            import traceback

            traceback.print_exc()
            table_summary = ""

        lines = content.split("\n")
        # Initialize an empty list to hold the parsed bullet points
        bullet_points = []
        # Loop through each line
        bullet_point_id = 1
        for line in lines:
            # If the line starts with '+', it is a bullet point
            if line.startswith("+"):
                # Remove the '+ ' from the start of the line and add it to the list
                bullet_points.append(f"{bullet_point_id}. " + line[1:].strip().strip('"'))
                bullet_point_id += 1
        return table_summary, "\n".join(bullet_points)


DataSummaryExecutor

from typing import Any, Dict, Tuple, Union
from abc import ABC

from langchain.base_language import BaseLanguageModel
from langchain import PromptTemplate

from real_agents.adapters.callbacks.executor_streaming import ExecutorStreamingChainHandler
from real_agents.adapters.data_model import DatabaseDataModel, TableDataModel, ImageDataModel
from real_agents.adapters.llm import LLMChain

class DataSummaryExecutor(ABC):
    tool_name = "DataProfiling"

    def _intelligent_summary(self, grounding_source: ImageDataModel, num_insights: int, llm: BaseLanguageModel) -> str:
        """Use LLM to generate data summary."""
        pass

class TableSummaryExecutor(DataSummaryExecutor):
    SUMMARY_PROMPT_TEMPLATE = """
{table_info}

Provide a succinct yet meaningful summary of the table with less than 20 words, encapsulating its essence beyond just enumerating the columns. Please ensure your summary is a complete sentence and include it within <summary></summary> tags."
Note the table actually far more rows than shown above, so you MUST NOT make any rash conclusions based on the shown table rows or cells."
Then provide {num_insights} insightful and interesting suggestions in natural language that users can directly say to analyze the table. The suggestions should be able to be solved by python/sql."
The final results should be markdown '+' bullet point list, e.g., + The first suggestion.

Begin."
"""
    stream_handler = ExecutorStreamingChainHandler()

    def run(
        self,
        grounding_source: Union[TableDataModel, DatabaseDataModel],
        llm: BaseLanguageModel,
        use_intelligent_summary: bool = True,


TableSummaryExecutor

from typing import Any, Dict, Tuple, Union
from abc import ABC

from langchain.base_language import BaseLanguageModel
from langchain import PromptTemplate

from real_agents.adapters.callbacks.executor_streaming import ExecutorStreamingChainHandler
from real_agents.adapters.data_model import DatabaseDataModel, TableDataModel, ImageDataModel
from real_agents.adapters.llm import LLMChain

class DataSummaryExecutor(ABC):
    tool_name = "DataProfiling"

    def _intelligent_summary(self, grounding_source: ImageDataModel, num_insights: int, llm: BaseLanguageModel) -> str:
        """Use LLM to generate data summary."""
        pass

class TableSummaryExecutor(DataSummaryExecutor):
    SUMMARY_PROMPT_TEMPLATE = """
{table_info}

Provide a succinct yet meaningful summary of the table with less than 20 words, encapsulating its essence beyond just enumerating the columns. Please ensure your summary is a complete sentence and include it within  tags."
Note the table actually far more rows than shown above, so you MUST NOT make any rash conclusions based on the shown table rows or cells."
Then provide {num_insights} insightful and interesting suggestions in natural language that users can directly say to analyze the table. The suggestions should be able to be solved by python/sql."
The final results should be markdown '+' bullet point list, e.g., + The first suggestion.

Begin."
"""
    stream_handler = ExecutorStreamingChainHandler()

    def run(
        self,
        grounding_source: Union[TableDataModel, DatabaseDataModel],
        llm: BaseLanguageModel,
        use_intelligent_summary: bool = True,
        num_insights: int = 3,
    ) -> Dict[str, Any]:
        summary = ""
        if isinstance(grounding_source, TableDataModel):
            df = grounding_source.raw_data
            df_name = grounding_source.raw_data_name
            # Basic summary
            summary += f"Your table {df_name} contains {df.shape[0]} rows and {df.shape[1]} columns. "

            null_count = df.isnull().sum().sum()  # Get total number of null values
            unique_values_avg = df.nunique().mean()  # Get average number of unique values

            summary += f"On average, each column has about {unique_values_avg:.0f} unique values. "
            if null_count > 0:
                summary += f"Watch out, there are {null_count} missing values in your data. "
            else:
                summary += "Good news, no missing values in your data. "

            # Intelligent summary
            if use_intelligent_summary:
                intelligent_summary = self._intelligent_summary(
                    grounding_source,
                    num_insights=num_insights,
                    llm=llm,
                )
                table_summary, suggestions = self._parse_output(intelligent_summary)
                summary += table_summary
                summary += "\n" + "Here are some additional insights to enhance your understanding of the table."
                summary += "\n" + suggestions

            for stream_token in summary.split(" "):
                self.stream_handler.on_llm_new_token(stream_token)

        elif isinstance(grounding_source, DatabaseDataModel):
            # TODO: Convert to df or use SQL query for basic summary
            raise NotImplementedError("DatabaseDataModel is not supported yet.")
        else:
            raise ValueError(f"Unsupported grounding source type: {type(grounding_source)}")
        return summary

    def _intelligent_summary(
        self, grounding_source: Union[TableDataModel, DatabaseDataModel], num_insights: int, llm: BaseLanguageModel
    ) -> str:
        """Use LLM to generate data summary."""
        summary_prompt_template = PromptTemplate(
            input_variables=["table_info", "num_insights"],
            template=self.SUMMARY_PROMPT_TEMPLATE,
        )
        method = LLMChain(llm=llm, prompt=summary_prompt_template)
        result = method.run({"table_info": grounding_source.get_llm_side_data(), "num_insights": num_insights})
        return result

    def _parse_output(self, content: str) -> Tuple[str, str]:
        """Parse the output of the LLM to get the data summary."""
        from bs4 import BeautifulSoup

        # Using 'html.parser' to parse the content
        soup = BeautifulSoup(content, "html.parser")
        # Parsing the tag and summary contents
        try:
            table_summary = soup.find("summary").text
        except Exception:
            import traceback

            traceback.print_exc()
            table_summary = ""

        lines = content.split("\n")
        # Initialize an empty list to hold the parsed bullet points
        bullet_points = []
        # Loop through each line
        bullet_point_id = 1
        for line in lines:
            # If the line starts with '+', it is a bullet point
            if line.startswith("+"):
                # Remove the '+ ' from the start of the line and add it to the list
                bullet_points.append(f"{bullet_point_id}. " + line[1:].strip().strip('"'))
                bullet_point_id += 1
        return table_summary, "\n".join(bullet_points)

class ImageSummaryExecutor(DataSummaryExecutor):
    SUMMARY_PROMPT_TEMPLATE = """
{img_info}

Provide a succinct summary of the uploaded file with less than 20 words. Please ensure your summary is a complete sentence and include it within  tags. For image, just show its name is basically enough."
Then provide {num_insights} very simple and basic suggestions in natural language about further processing with the data. The suggestions should be able to be solved by python(e.g., grayscale, rescale, rotation, etc). The final results should be markdown '+' bullet point list, e.g., + The first suggestion."

Begin.
"""
    stream_handler = ExecutorStreamingChainHandler()

    def run(
        self,
        grounding_source: ImageDataModel,
        llm: BaseLanguageModel,
        use_intelligent_summary: bool = True,
        num_insights: int = 3,
    ) -> Dict[str, Any]:


ImageSummaryExecutor

        except Exception:
            import traceback

            traceback.print_exc()
            table_summary = ""

        lines = content.split("\n")
        # Initialize an empty list to hold the parsed bullet points
        bullet_points = []
        # Loop through each line
        bullet_point_id = 1
        for line in lines:
            # If the line starts with '+', it is a bullet point
            if line.startswith("+"):
                # Remove the '+ ' from the start of the line and add it to the list
                bullet_points.append(f"{bullet_point_id}. " + line[1:].strip().strip('"'))
                bullet_point_id += 1
        return table_summary, "\n".join(bullet_points)

class ImageSummaryExecutor(DataSummaryExecutor):
    SUMMARY_PROMPT_TEMPLATE = """
{img_info}

Provide a succinct summary of the uploaded file with less than 20 words. Please ensure your summary is a complete sentence and include it within <summary></summary> tags. For image, just show its name is basically enough."
Then provide {num_insights} very simple and basic suggestions in natural language about further processing with the data. The suggestions should be able to be solved by python(e.g., grayscale, rescale, rotation, etc). The final results should be markdown '+' bullet point list, e.g., + The first suggestion."

Begin.
"""
    stream_handler = ExecutorStreamingChainHandler()

    def run(
        self,
        grounding_source: ImageDataModel,
        llm: BaseLanguageModel,
        use_intelligent_summary: bool = True,
        num_insights: int = 3,
    ) -> Dict[str, Any]:
        summary = ""
        if isinstance(grounding_source, ImageDataModel):
            # Basic summary
            raw_data = grounding_source.raw_data
            img_size, img_mode, img_format = raw_data["size"], raw_data["mode"], raw_data["format"]
            summary += f"Your image **{grounding_source.simple_filename}** is a {img_size[0]}x{img_size[1]} {img_mode} image in {img_format} format.\n"

            # Intelligent summary
            if use_intelligent_summary:
                intelligent_summary = self._intelligent_summary(
                    grounding_source,
                    num_insights=num_insights,
                    llm=llm,
                )
                _, suggestions = self._parse_output(intelligent_summary)
                summary += "\n" + "Here are some additional insights to enhance your understanding of the image"
                summary += "\n" + suggestions

            for stream_token in summary.split(" "):
                self.stream_handler.on_llm_new_token(stream_token)
        else:
            raise ValueError(f"Unsupported data summary for grounding source type: {type(grounding_source)}")
        return summary

    def _intelligent_summary(self, grounding_source: ImageDataModel, num_insights: int, llm: BaseLanguageModel) -> str:
        """Use LLM to generate data summary."""
        summary_prompt_template = PromptTemplate(
            input_variables=["img_info", "num_insights"],
            template=self.SUMMARY_PROMPT_TEMPLATE,
        )
        method = LLMChain(llm=llm, prompt=summary_prompt_template)
        result = method.run({"img_info": grounding_source.get_llm_side_data(), "num_insights": num_insights})
        return result

    def _parse_output(self, content: str) -> Tuple[str, str]:
        """Parse the output of the LLM to get the data summary."""
        from bs4 import BeautifulSoup

        # Using 'html.parser' to parse the content
        soup = BeautifulSoup(content, "html.parser")
        # Parsing the tag and summary contents
        try:
            table_summary = soup.find("summary").text
        except Exception:
            import traceback

            traceback.print_exc()
            table_summary = ""

        lines = content.split("\n")
        # Initialize an empty list to hold the parsed bullet points
        bullet_points = []
        # Loop through each line
        bullet_point_id = 1
        for line in lines:
            # If the line starts with '+', it is a bullet point
            if line.startswith("+"):
                # Remove the '+ ' from the start of the line and add it to the list
                bullet_points.append(f"{bullet_point_id}. " + line[1:].strip().strip('"'))
                bullet_point_id += 1
        return table_summary, "\n".join(bullet_points)


            img_size, img_mode, img_format = raw_data["size"], raw_data["mode"], raw_data["format"]
            summary += f"Your image **{grounding_source.simple_filename}** is a {img_size[0]}x{img_size[1]} {img_mode} image in {img_format} format.\n"

            # Intelligent summary
            if use_intelligent_summary:
                intelligent_summary = self._intelligent_summary(
                    grounding_source,
                    num_insights=num_insights,
                    llm=llm,
                )
                _, suggestions = self._parse_output(intelligent_summary)
                summary += "\n" + "Here are some additional insights to enhance your understanding of the image"
                summary += "\n" + suggestions

            for stream_token in summary.split(" "):
                self.stream_handler.on_llm_new_token(stream_token)
        else:
            raise ValueError(f"Unsupported data summary for grounding source type: {type(grounding_source)}")
        return summary

    def _intelligent_summary(self, grounding_source: ImageDataModel, num_insights: int, llm: BaseLanguageModel) -> str:
        """Use LLM to generate data summary."""
        summary_prompt_template = PromptTemplate(
            input_variables=["img_info", "num_insights"],
            template=self.SUMMARY_PROMPT_TEMPLATE,
        )
        method = LLMChain(llm=llm, prompt=summary_prompt_template)
        result = method.run({"img_info": grounding_source.get_llm_side_data(), "num_insights": num_insights})
        return result

    def _parse_output(self, content: str) -> Tuple[str, str]:
        """Parse the output of the LLM to get the data summary."""
        from bs4 import BeautifulSoup

        # Using 'html.parser' to parse the content
        soup = BeautifulSoup(content, "html.parser")
        # Parsing the tag and summary contents
        try:
            table_summary = soup.find("summary").text
        except Exception:
            import traceback

            traceback.print_exc()
            table_summary = ""

        lines = content.split("\n")
        # Initialize an empty list to hold the parsed bullet points
        bullet_points = []
        # Loop through each line


run

        for line in lines:
            # If the line starts with '+', it is a bullet point
            if line.startswith("+"):
                # Remove the '+ ' from the start of the line and add it to the list
                bullet_points.append(f"{bullet_point_id}. " + line[1:].strip().strip('"'))
                bullet_point_id += 1
        return table_summary, "\n".join(bullet_points)

class ImageSummaryExecutor(DataSummaryExecutor):
    SUMMARY_PROMPT_TEMPLATE = """
{img_info}

Provide a succinct summary of the uploaded file with less than 20 words. Please ensure your summary is a complete sentence and include it within <summary></summary> tags. For image, just show its name is basically enough."
Then provide {num_insights} very simple and basic suggestions in natural language about further processing with the data. The suggestions should be able to be solved by python(e.g., grayscale, rescale, rotation, etc). The final results should be markdown '+' bullet point list, e.g., + The first suggestion."

Begin.
"""
    stream_handler = ExecutorStreamingChainHandler()

    def run(
        self,
        grounding_source: ImageDataModel,
        llm: BaseLanguageModel,
        use_intelligent_summary: bool = True,
        num_insights: int = 3,
    ) -> Dict[str, Any]:
        summary = ""
        if isinstance(grounding_source, ImageDataModel):
            # Basic summary
            raw_data = grounding_source.raw_data
            img_size, img_mode, img_format = raw_data["size"], raw_data["mode"], raw_data["format"]
            summary += f"Your image **{grounding_source.simple_filename}** is a {img_size[0]}x{img_size[1]} {img_mode} image in {img_format} format.\n"

            # Intelligent summary
            if use_intelligent_summary:
                intelligent_summary = self._intelligent_summary(
                    grounding_source,
                    num_insights=num_insights,
                    llm=llm,
                )
                _, suggestions = self._parse_output(intelligent_summary)
                summary += "\n" + "Here are some additional insights to enhance your understanding of the image"
                summary += "\n" + suggestions

            for stream_token in summary.split(" "):
                self.stream_handler.on_llm_new_token(stream_token)
        else:
            raise ValueError(f"Unsupported data summary for grounding source type: {type(grounding_source)}")
        return summary

    def _intelligent_summary(self, grounding_source: ImageDataModel, num_insights: int, llm: BaseLanguageModel) -> str:
        """Use LLM to generate data summary."""
        summary_prompt_template = PromptTemplate(
            input_variables=["img_info", "num_insights"],
            template=self.SUMMARY_PROMPT_TEMPLATE,
        )
        method = LLMChain(llm=llm, prompt=summary_prompt_template)
        result = method.run({"img_info": grounding_source.get_llm_side_data(), "num_insights": num_insights})
        return result

    def _parse_output(self, content: str) -> Tuple[str, str]:
        """Parse the output of the LLM to get the data summary."""
        from bs4 import BeautifulSoup

        # Using 'html.parser' to parse the content
        soup = BeautifulSoup(content, "html.parser")
        # Parsing the tag and summary contents
        try:
            table_summary = soup.find("summary").text


                _, suggestions = self._parse_output(intelligent_summary)
                summary += "\n" + "Here are some additional insights to enhance your understanding of the image"
                summary += "\n" + suggestions

            for stream_token in summary.split(" "):
                self.stream_handler.on_llm_new_token(stream_token)
        else:
            raise ValueError(f"Unsupported data summary for grounding source type: {type(grounding_source)}")
        return summary

    def _intelligent_summary(self, grounding_source: ImageDataModel, num_insights: int, llm: BaseLanguageModel) -> str:
        """Use LLM to generate data summary."""
        summary_prompt_template = PromptTemplate(
            input_variables=["img_info", "num_insights"],
            template=self.SUMMARY_PROMPT_TEMPLATE,
        )
        method = LLMChain(llm=llm, prompt=summary_prompt_template)
        result = method.run({"img_info": grounding_source.get_llm_side_data(), "num_insights": num_insights})
        return result

    def _parse_output(self, content: str) -> Tuple[str, str]:
        """Parse the output of the LLM to get the data summary."""
        from bs4 import BeautifulSoup

        # Using 'html.parser' to parse the content
        soup = BeautifulSoup(content, "html.parser")
        # Parsing the tag and summary contents
        try:
            table_summary = soup.find("summary").text
        except Exception:
            import traceback

            traceback.print_exc()
            table_summary = ""

        lines = content.split("\n")
        # Initialize an empty list to hold the parsed bullet points
        bullet_points = []
        # Loop through each line
        bullet_point_id = 1
        for line in lines:
            # If the line starts with '+', it is a bullet point
            if line.startswith("+"):
                # Remove the '+ ' from the start of the line and add it to the list
                bullet_points.append(f"{bullet_point_id}. " + line[1:].strip().strip('"'))
                bullet_point_id += 1
        return table_summary, "\n".join(bullet_points)


"""Chain for interacting with SQL Database."""
from __future__ import annotations

from typing import Any, Dict, List, Optional
from pydantic import BaseModel, Extra, Field
from loguru import logger

from langchain.base_language import BaseLanguageModel
from langchain.callbacks.manager import CallbackManagerForChainRun
from langchain.chains.base import Chain
from langchain import BasePromptTemplate, FewShotPromptTemplate

from real_agents.data_agent.evaluation.sql_evaluator import SQLEvaluator
from real_agents.adapters.schema import SQLDatabase
from real_agents.adapters.memory import ReadOnlySharedStringMemory
from real_agents.data_agent.sql.prompt import (
    EXAMPLE_PROMPT,
    FEW_SHOT_INPUT_VARIABLES,
    FEW_SHOT_PREFIX,
    FEW_SHOT_SUFFIX,
    PROMPT,
)
from real_agents.adapters.llm import LLMChain
from real_agents.adapters.data_model import MessageDataModel

class SQLDatabaseChain(Chain, BaseModel):
    """Chain for interacting with SQL Database"""

    llm: BaseLanguageModel
    """LLM wrapper to use."""
    database: SQLDatabase = Field(exclude=True)
    """SQL Database to connect to."""
    example_selector: Any = None
    """Example selector to select few-shot in-context exemplars."""
    memory: Optional[ReadOnlySharedStringMemory] = None
    """Shared memory."""
    prompt: BasePromptTemplate = PROMPT
    """Prompt to use to translate natural language to SQL."""
    input_key: str = "user_intent"  #: :meta private:
    output_key: str = "result"  #: :meta private:
    return_intermediate_steps: bool = False
    """Whether or not to return the intermediate steps along with the final answer."""
    return_direct: bool = False
    """Whether or not to return the result of querying the SQL table directly."""

    class Config:
        """Configuration for this pydantic object."""

        extra = Extra.forbid
        arbitrary_types_allowed = True

    @property
    def input_keys(self) -> List[str]:
        """Return the singular input key.
        :meta private:
        """
        return [self.input_key]

    @property
    def output_keys(self) -> List[str]:
        """Return the singular output key.
        :meta private:
        """
        if not self.return_intermediate_steps:
            return [self.output_key]
        else:
            # return [self.output_key, "intermediate_steps", "binder_steps"]
            return [self.output_key, "intermediate_steps"]

    def _call(
        self,
        inputs: Dict[str, str],
        run_manager: Optional[CallbackManagerForChainRun] = None,
    ) -> Dict[str, str]:
        logger.bind(msg_head="SQLChain inputs").trace(inputs)

        _run_manager = run_manager or CallbackManagerForChainRun.get_noop_manager()
        if self.example_selector is not None:
            self.prompt = FewShotPromptTemplate(
                example_selector=self.example_selector,
                example_prompt=EXAMPLE_PROMPT,
                prefix=FEW_SHOT_PREFIX,
                suffix=FEW_SHOT_SUFFIX,
                input_variables=FEW_SHOT_INPUT_VARIABLES,
            )
        llm_chain = LLMChain(llm=self.llm, prompt=self.prompt)
        input_text = f"{inputs[self.input_key]} \nSQLQuery:"
        _run_manager.on_text(input_text, verbose=self.verbose)

        # If not present, then defaults to None which is all tables.
        table_names_to_use = inputs.get("table_names_to_use")
        table_info = self.database.get_table_info(table_names=table_names_to_use)
        llm_inputs = {
            "question": input_text,
            "dialect": self.database.dialect,
            "table_info": table_info,
            "chat_history": "",
            "stop": ["\nSQLResult:"],
        }

        # Load memory into chat history
        if self.memory is not None:
            llm_inputs["chat_history"] = self.memory.load_memory_variables({})["chat_history"]
            llm_inputs["chat_history"] = MessageDataModel.extract_code_for_sql_tool(llm_inputs["chat_history"])
        sql_cmd = llm_chain.predict(**llm_inputs)
        # TODO: Move this post-processing to a post-process function
        sql_cmd = sql_cmd.replace("\n", " ")
        if sql_cmd.endswith('"') and sql_cmd.startswith('"'):
            sql_cmd = sql_cmd.strip('"')
        if sql_cmd.endswith("'") and sql_cmd.startswith("'"):
            sql_cmd = sql_cmd.strip("'")

        logger.bind(msg_head="SQLChain generate program").trace(sql_cmd)

        # Call SQL/binder evaluator to execute the SQL command
        sql_evaluator = SQLEvaluator()
        result = sql_evaluator.run(sql_cmd, self.database)

        logger.bind(msg_head="SQLChain execution result").trace(result)

        # If return direct, we just set the final result equal to the sql query
        if self.return_direct:
            final_result = result
        else:
            input_text += f"{sql_cmd}\nSQLResult: {result}\nAnswer:"
            llm_inputs["input"] = input_text
            final_result = llm_chain.predict(**llm_inputs)
            _run_manager.on_text(final_result, color="green", verbose=self.verbose)
        chain_result: Dict[str, Any] = {self.output_key: final_result}
        if self.return_intermediate_steps:
            chain_result["intermediate_steps"] = sql_cmd
        return chain_result

    @property
    def _chain_type(self) -> str:
        return "sql_database_chain"


SQLDatabaseChain

from langchain.base_language import BaseLanguageModel
from langchain.callbacks.manager import CallbackManagerForChainRun
from langchain.chains.base import Chain
from langchain import BasePromptTemplate, FewShotPromptTemplate

from real_agents.data_agent.evaluation.sql_evaluator import SQLEvaluator
from real_agents.adapters.schema import SQLDatabase
from real_agents.adapters.memory import ReadOnlySharedStringMemory
from real_agents.data_agent.sql.prompt import (
    EXAMPLE_PROMPT,
    FEW_SHOT_INPUT_VARIABLES,
    FEW_SHOT_PREFIX,
    FEW_SHOT_SUFFIX,
    PROMPT,
)
from real_agents.adapters.llm import LLMChain
from real_agents.adapters.data_model import MessageDataModel

class SQLDatabaseChain(Chain, BaseModel):
    """Chain for interacting with SQL Database"""

    llm: BaseLanguageModel
    """LLM wrapper to use."""
    database: SQLDatabase = Field(exclude=True)
    """SQL Database to connect to."""
    example_selector: Any = None
    """Example selector to select few-shot in-context exemplars."""
    memory: Optional[ReadOnlySharedStringMemory] = None
    """Shared memory."""
    prompt: BasePromptTemplate = PROMPT
    """Prompt to use to translate natural language to SQL."""
    input_key: str = "user_intent"  #: :meta private:
    output_key: str = "result"  #: :meta private:
    return_intermediate_steps: bool = False
    """Whether or not to return the intermediate steps along with the final answer."""
    return_direct: bool = False
    """Whether or not to return the result of querying the SQL table directly."""

    class Config:
        """Configuration for this pydantic object."""

        extra = Extra.forbid
        arbitrary_types_allowed = True

    @property
    def input_keys(self) -> List[str]:
        """Return the singular input key.
        :meta private:
        """
        return [self.input_key]

    @property
    def output_keys(self) -> List[str]:
        """Return the singular output key.
        :meta private:
        """
        if not self.return_intermediate_steps:
            return [self.output_key]
        else:
            # return [self.output_key, "intermediate_steps", "binder_steps"]
            return [self.output_key, "intermediate_steps"]

    def _call(
        self,
        inputs: Dict[str, str],
        run_manager: Optional[CallbackManagerForChainRun] = None,
    ) -> Dict[str, str]:
        logger.bind(msg_head="SQLChain inputs").trace(inputs)

        _run_manager = run_manager or CallbackManagerForChainRun.get_noop_manager()
        if self.example_selector is not None:
            self.prompt = FewShotPromptTemplate(
                example_selector=self.example_selector,
                example_prompt=EXAMPLE_PROMPT,
                prefix=FEW_SHOT_PREFIX,
                suffix=FEW_SHOT_SUFFIX,
                input_variables=FEW_SHOT_INPUT_VARIABLES,
            )
        llm_chain = LLMChain(llm=self.llm, prompt=self.prompt)
        input_text = f"{inputs[self.input_key]} \nSQLQuery:"
        _run_manager.on_text(input_text, verbose=self.verbose)

        # If not present, then defaults to None which is all tables.
        table_names_to_use = inputs.get("table_names_to_use")
        table_info = self.database.get_table_info(table_names=table_names_to_use)
        llm_inputs = {
            "question": input_text,
            "dialect": self.database.dialect,
            "table_info": table_info,
            "chat_history": "",
            "stop": ["\nSQLResult:"],
        }

        # Load memory into chat history
        if self.memory is not None:
            llm_inputs["chat_history"] = self.memory.load_memory_variables({})["chat_history"]
            llm_inputs["chat_history"] = MessageDataModel.extract_code_for_sql_tool(llm_inputs["chat_history"])
        sql_cmd = llm_chain.predict(**llm_inputs)
        # TODO: Move this post-processing to a post-process function
        sql_cmd = sql_cmd.replace("\n", " ")
        if sql_cmd.endswith('"') and sql_cmd.startswith('"'):
            sql_cmd = sql_cmd.strip('"')
        if sql_cmd.endswith("'") and sql_cmd.startswith("'"):
            sql_cmd = sql_cmd.strip("'")

        logger.bind(msg_head="SQLChain generate program").trace(sql_cmd)

        # Call SQL/binder evaluator to execute the SQL command
        sql_evaluator = SQLEvaluator()
        result = sql_evaluator.run(sql_cmd, self.database)

        logger.bind(msg_head="SQLChain execution result").trace(result)

        # If return direct, we just set the final result equal to the sql query
        if self.return_direct:
            final_result = result
        else:
            input_text += f"{sql_cmd}\nSQLResult: {result}\nAnswer:"
            llm_inputs["input"] = input_text
            final_result = llm_chain.predict(**llm_inputs)
            _run_manager.on_text(final_result, color="green", verbose=self.verbose)
        chain_result: Dict[str, Any] = {self.output_key: final_result}
        if self.return_intermediate_steps:
            chain_result["intermediate_steps"] = sql_cmd
        return chain_result

    @property
    def _chain_type(self) -> str:
        return "sql_database_chain"


Config

class SQLDatabaseChain(Chain, BaseModel):
    """Chain for interacting with SQL Database"""

    llm: BaseLanguageModel
    """LLM wrapper to use."""
    database: SQLDatabase = Field(exclude=True)
    """SQL Database to connect to."""
    example_selector: Any = None
    """Example selector to select few-shot in-context exemplars."""
    memory: Optional[ReadOnlySharedStringMemory] = None
    """Shared memory."""
    prompt: BasePromptTemplate = PROMPT
    """Prompt to use to translate natural language to SQL."""
    input_key: str = "user_intent"  #: :meta private:
    output_key: str = "result"  #: :meta private:
    return_intermediate_steps: bool = False
    """Whether or not to return the intermediate steps along with the final answer."""
    return_direct: bool = False
    """Whether or not to return the result of querying the SQL table directly."""

    class Config:
        """Configuration for this pydantic object."""

        extra = Extra.forbid
        arbitrary_types_allowed = True

    @property
    def input_keys(self) -> List[str]:
        """Return the singular input key.
        :meta private:
        """
        return [self.input_key]

    @property
    def output_keys(self) -> List[str]:
        """Return the singular output key.
        :meta private:
        """
        if not self.return_intermediate_steps:
            return [self.output_key]
        else:
            # return [self.output_key, "intermediate_steps", "binder_steps"]
            return [self.output_key, "intermediate_steps"]

    def _call(


    example_selector: Any = None
    """Example selector to select few-shot in-context exemplars."""
    memory: Optional[ReadOnlySharedStringMemory] = None
    """Shared memory."""
    prompt: BasePromptTemplate = PROMPT
    """Prompt to use to translate natural language to SQL."""
    input_key: str = "user_intent"  #: :meta private:
    output_key: str = "result"  #: :meta private:
    return_intermediate_steps: bool = False
    """Whether or not to return the intermediate steps along with the final answer."""
    return_direct: bool = False
    """Whether or not to return the result of querying the SQL table directly."""

    class Config:
        """Configuration for this pydantic object."""

        extra = Extra.forbid
        arbitrary_types_allowed = True

    @property
    def input_keys(self) -> List[str]:
        """Return the singular input key.
        :meta private:
        """
        return [self.input_key]

    @property
    def output_keys(self) -> List[str]:
        """Return the singular output key.
        :meta private:
        """
        if not self.return_intermediate_steps:
            return [self.output_key]
        else:
            # return [self.output_key, "intermediate_steps", "binder_steps"]
            return [self.output_key, "intermediate_steps"]

    def _call(
        self,
        inputs: Dict[str, str],
        run_manager: Optional[CallbackManagerForChainRun] = None,
    ) -> Dict[str, str]:
        logger.bind(msg_head="SQLChain inputs").trace(inputs)

        _run_manager = run_manager or CallbackManagerForChainRun.get_noop_manager()


    output_key: str = "result"  #: :meta private:
    return_intermediate_steps: bool = False
    """Whether or not to return the intermediate steps along with the final answer."""
    return_direct: bool = False
    """Whether or not to return the result of querying the SQL table directly."""

    class Config:
        """Configuration for this pydantic object."""

        extra = Extra.forbid
        arbitrary_types_allowed = True

    @property
    def input_keys(self) -> List[str]:
        """Return the singular input key.
        :meta private:
        """
        return [self.input_key]

    @property
    def output_keys(self) -> List[str]:
        """Return the singular output key.
        :meta private:
        """
        if not self.return_intermediate_steps:
            return [self.output_key]
        else:
            # return [self.output_key, "intermediate_steps", "binder_steps"]
            return [self.output_key, "intermediate_steps"]

    def _call(
        self,
        inputs: Dict[str, str],
        run_manager: Optional[CallbackManagerForChainRun] = None,
    ) -> Dict[str, str]:
        logger.bind(msg_head="SQLChain inputs").trace(inputs)

        _run_manager = run_manager or CallbackManagerForChainRun.get_noop_manager()
        if self.example_selector is not None:
            self.prompt = FewShotPromptTemplate(
                example_selector=self.example_selector,
                example_prompt=EXAMPLE_PROMPT,
                prefix=FEW_SHOT_PREFIX,
                suffix=FEW_SHOT_SUFFIX,
                input_variables=FEW_SHOT_INPUT_VARIABLES,
            )
        llm_chain = LLMChain(llm=self.llm, prompt=self.prompt)
        input_text = f"{inputs[self.input_key]} \nSQLQuery:"
        _run_manager.on_text(input_text, verbose=self.verbose)


        arbitrary_types_allowed = True

    @property
    def input_keys(self) -> List[str]:
        """Return the singular input key.
        :meta private:
        """
        return [self.input_key]

    @property
    def output_keys(self) -> List[str]:
        """Return the singular output key.
        :meta private:
        """
        if not self.return_intermediate_steps:
            return [self.output_key]
        else:
            # return [self.output_key, "intermediate_steps", "binder_steps"]
            return [self.output_key, "intermediate_steps"]

    def _call(
        self,
        inputs: Dict[str, str],
        run_manager: Optional[CallbackManagerForChainRun] = None,
    ) -> Dict[str, str]:
        logger.bind(msg_head="SQLChain inputs").trace(inputs)

        _run_manager = run_manager or CallbackManagerForChainRun.get_noop_manager()
        if self.example_selector is not None:
            self.prompt = FewShotPromptTemplate(
                example_selector=self.example_selector,
                example_prompt=EXAMPLE_PROMPT,
                prefix=FEW_SHOT_PREFIX,
                suffix=FEW_SHOT_SUFFIX,
                input_variables=FEW_SHOT_INPUT_VARIABLES,
            )
        llm_chain = LLMChain(llm=self.llm, prompt=self.prompt)
        input_text = f"{inputs[self.input_key]} \nSQLQuery:"
        _run_manager.on_text(input_text, verbose=self.verbose)

        # If not present, then defaults to None which is all tables.
        table_names_to_use = inputs.get("table_names_to_use")
        table_info = self.database.get_table_info(table_names=table_names_to_use)
        llm_inputs = {
            "question": input_text,
            "dialect": self.database.dialect,
            "table_info": table_info,
            "chat_history": "",
            "stop": ["\nSQLResult:"],
        }

        # Load memory into chat history
        if self.memory is not None:
            llm_inputs["chat_history"] = self.memory.load_memory_variables({})["chat_history"]
            llm_inputs["chat_history"] = MessageDataModel.extract_code_for_sql_tool(llm_inputs["chat_history"])
        sql_cmd = llm_chain.predict(**llm_inputs)
        # TODO: Move this post-processing to a post-process function
        sql_cmd = sql_cmd.replace("\n", " ")
        if sql_cmd.endswith('"') and sql_cmd.startswith('"'):
            sql_cmd = sql_cmd.strip('"')
        if sql_cmd.endswith("'") and sql_cmd.startswith("'"):
            sql_cmd = sql_cmd.strip("'")

        logger.bind(msg_head="SQLChain generate program").trace(sql_cmd)

        # Call SQL/binder evaluator to execute the SQL command
        sql_evaluator = SQLEvaluator()
        result = sql_evaluator.run(sql_cmd, self.database)

        logger.bind(msg_head="SQLChain execution result").trace(result)

        # If return direct, we just set the final result equal to the sql query
        if self.return_direct:
            final_result = result
        else:
            input_text += f"{sql_cmd}\nSQLResult: {result}\nAnswer:"
            llm_inputs["input"] = input_text
            final_result = llm_chain.predict(**llm_inputs)
            _run_manager.on_text(final_result, color="green", verbose=self.verbose)
        chain_result: Dict[str, Any] = {self.output_key: final_result}
        if self.return_intermediate_steps:
            chain_result["intermediate_steps"] = sql_cmd
        return chain_result

    @property
    def _chain_type(self) -> str:
        return "sql_database_chain"


        # Call SQL/binder evaluator to execute the SQL command
        sql_evaluator = SQLEvaluator()
        result = sql_evaluator.run(sql_cmd, self.database)

        logger.bind(msg_head="SQLChain execution result").trace(result)

        # If return direct, we just set the final result equal to the sql query
        if self.return_direct:
            final_result = result
        else:
            input_text += f"{sql_cmd}\nSQLResult: {result}\nAnswer:"
            llm_inputs["input"] = input_text
            final_result = llm_chain.predict(**llm_inputs)
            _run_manager.on_text(final_result, color="green", verbose=self.verbose)
        chain_result: Dict[str, Any] = {self.output_key: final_result}
        if self.return_intermediate_steps:
            chain_result["intermediate_steps"] = sql_cmd
        return chain_result

    @property
    def _chain_type(self) -> str:
        return "sql_database_chain"


# flake8: noqa
from langchain import PromptTemplate

# Text-to-sql prompt
_DEFAULT_TEMPLATE = """Here are chat histories you may refer to, maybe empty.
{chat_history}

Given an input question, first create a syntactically correct {dialect} query to run, then look at the results of the query and return the answer.
Never query for all the columns from a specific table, only ask for a the few relevant columns given the question.
Pay attention to use only the column names that you can see in the schema description. Be careful to not query for columns that do not exist. Also, remember to wrap the table names in double quotes.
Use the following format:
Question: "Question here"
SQLQuery: "SQL Query to run"
SQLResult: "Result of the SQLQuery"
Answer: "Final answer here"
Only use the tables listed below.
{table_info}
Question: {question}"""
PROMPT = PromptTemplate(
    input_variables=["chat_history", "question", "table_info", "dialect"],
    template=_DEFAULT_TEMPLATE,
)

# Few-shot text-to-sql prompt
FEW_SHOT_PREFIX = """Given an input question, first create a syntactically correct {dialect} query to run, then look at the results of the query and return the answer. Unless the user specifies in his question a specific number of examples he wishes to obtain, always limit your query to at most {top_k} results. You can order the results by a relevant column to return the most interesting examples in the database.
Never query for all the columns from a specific table, only ask for a the few relevant columns given the question.
Pay attention to use only the column names that you can see in the schema description. Be careful to not query for columns that do not exist. Also, pay attention to which column is in which table.
Use the following format:
Question: "Question here"
SQLQuery: "SQL Query to run"
SQLResult: "Result of the SQLQuery"
Answer: "Final answer here"
Here are some examples you can follow:"""
EXAMPLE_PROMPT_TEMPLATE = """{table_info}\nQuestion: {question}\nSQLQuery: {query}"""
EXAMPLE_PROMPT = PromptTemplate(
    input_variables=["table_info", "question", "query"],
    template=EXAMPLE_PROMPT_TEMPLATE,
)
FEW_SHOT_SUFFIX = """
User the tables listed below.
{table_info}
Question: {question}"""
FEW_SHOT_INPUT_VARIABLES = ["question", "table_info", "dialect", "top_k"]