chat

Output: The action you want to perform, and the extracted path or searched relevant datasets depending on your input.
""",
        ),
    }
    # Data profiling is not activated in agent
    IGNORE_TOOLS = ["DataProfiling"]
    # Activate tools according to the user selection
    tools = [tool_dict[lang["name"]] for lang in code_interpreter_languages]
    for tool in code_interpreter_tools:
        if tool["name"] not in IGNORE_TOOLS:
            tools.append(tool_dict[tool["name"]])

    # Build the chat agent with LLM and tools
    continue_model = llm_name if llm_name in NEED_CONTINUE_MODEL else None
    interaction_executor = initialize_agent(tools, llm, continue_model, memory=memory,
                                            verbose=True)
    return interaction_executor

@app.route("/api/chat", methods=["POST"])
def chat() -> Response | Dict:
    """Returns the chat response of data agent."""
    try:
        # Get request parameters
        request_json = request.get_json()
        user_id = request_json.pop("user_id", DEFAULT_USER_ID)
        chat_id = request_json["chat_id"]
        user_intent = request_json["user_intent"]
        parent_message_id = int(request_json["parent_message_id"])
        code_interpreter_languages = request_json.get("code_interpreter_languages", [])
        code_interpreter_tools = request_json.get("code_interpreter_tools", [])
        api_call = request_json.get("api_call", None)
        llm_name = request_json["llm_name"]
        temperature = request_json.get("temperature", 0.7)
        stop_words = ["[RESPONSE_BEGIN]", "TOOL RESPONSE"]
        kwargs = {
            "temperature": temperature,
            "stop": stop_words,
        }

        # Get language model
        stream_handler = AgentStreamingStdOutCallbackHandler()
        llm = get_llm(llm_name, **kwargs)

        logger.bind(user_id=user_id, chat_id=chat_id, api="/chat",
                    msg_head="Request json").debug(request_json)

        if api_call:
            # Load/init grounding source
            grounding_source_dict = grounding_source_pool.get_pool_info_with_id(user_id,
                                                                                chat_id,
                                                                                default_value={})

            # Find the mainstay message list from leaf to root
            activated_message_list = message_pool.get_activated_message_list(
                user_id, chat_id, default_value=list(),
                parent_message_id=parent_message_id
            )
            assert api_call["api_name"] == "DataProfiling"
            ai_message_id = message_id_register.add_variable("")
            file_node = api_call["args"]["activated_file"]

            folder = create_personal_folder(user_id)
            file_path = _get_file_path_from_node(folder, file_node)
            executor = get_data_summary_cls(file_path)()
            gs = grounding_source_dict[file_path]
            return stream_with_context(
                Response(
                    single_round_chat_with_executor(
                        executor,
                        user_intent=gs,
                        human_message_id=None,
                        ai_message_id=ai_message_id,
                        user_id=DEFAULT_USER_ID,
                        chat_id=api_call["args"]["chat_id"],
                        message_list=activated_message_list,
                        parent_message_id=api_call["args"]["parent_message_id"],
                        llm=llm,
                        app_type="copilot",
                    ),
                    content_type="application/json",
                )
            )
        else:
            # Load/init grounding source
            grounding_source_dict = grounding_source_pool.get_pool_info_with_id(user_id,
                                                                                chat_id,
                                                                                default_value={})
            # Build executor and run chat
            interaction_executor = create_interaction_executor(
                grounding_source_dict=grounding_source_dict,
                code_interpreter_languages=code_interpreter_languages,
                code_interpreter_tools=code_interpreter_tools,
                llm=llm,
                llm_name=llm_name,
                user_id=user_id,
                chat_id=chat_id,
                code_execution_mode=app.config["CODE_EXECUTION_MODE"],
            )
            # Find the mainstay message list from leaf to root
            activated_message_list = message_pool.get_activated_message_list(
                user_id, chat_id, default_value=list(),
                parent_message_id=parent_message_id
            )
            message_pool.load_agent_memory_from_list(interaction_executor.memory,
                                                     activated_message_list)
            human_message_id = message_id_register.add_variable(user_intent)
            ai_message_id = message_id_register.add_variable("")
            return stream_with_context(
                Response(
                    single_round_chat_with_agent_streaming(
                        interaction_executor=interaction_executor,
                        user_intent=user_intent,
                        human_message_id=human_message_id,
                        ai_message_id=ai_message_id,
                        user_id=user_id,
                        chat_id=chat_id,
                        message_list=activated_message_list,
                        parent_message_id=parent_message_id,
                        llm_name=llm_name,
                        stream_handler=stream_handler,
                        app_type="copilot"
                    ),
                    content_type="application/json",
                )
            )

    except Exception as e:
        try:
            logger.bind(user_id=user_id, chat_id=chat_id, api="/chat",
                        msg_head="Chat error").error(str(e))
            import traceback

            traceback.print_exc()
        except:
            # if user_id & chat_id not found, unauth err
            return Response(response=None, status=f"{UNAUTH} Invalid Authentication")
        return Response(response=None,
                        status=f"{OVERLOAD} Server is currently overloaded")


        An agent executor.

    """
    # Initialize Memory
    memory = ConversationReActBufferMemory(
        memory_key="chat_history", return_messages=True, llm=llm, max_token_limit=3500
    )
    read_only_memory = ReadOnlySharedStringMemory(memory=memory)

    # Initialize tools(executors)
    basic_chat_executor = ChatExecutor()
    python_code_generation_executor = CodeGenerationExecutor(
        programming_language="python", memory=read_only_memory)
    sql_code_generation_executor = CodeGenerationExecutor(programming_language="sql",
                                                          memory=read_only_memory)
    echart_code_generation_executor = CodeGenerationExecutor(
        programming_language="python", memory=read_only_memory, usage="echarts"
    )
    kaggle_data_loading_executor = KaggleDataLoadingExecutor()

    def run_python_code_builder(term: str) -> Union[Dict, DataModel]:
        try:
            # Only TableDataModel are allowed as input to python
            # input_grounding_source = [gs for _, gs in grounding_source_dict.items()
            # if isinstance(gs, TableDataModel)]
            input_grounding_source = [gs for gs in grounding_source_dict.values()]
            # Get the result
            results = python_code_generation_executor.run(
                user_intent=term,
                llm=llm,
                grounding_source=input_grounding_source,
                user_id=user_id,
                chat_id=chat_id,
                code_execution_mode=code_execution_mode,
                jupyter_kernel_pool=jupyter_kernel_pool,
            )

            logger.bind(msg_head=f"PythonCodeBuilder results({llm})").debug(results)

            if results["result"]["success"]:
                if results["result"]["result"] is not None:
                    raw_output = results["result"]["result"]
                elif results["result"]["stdout"] != "":
                    raw_output = results["result"]["stdout"]
                else:
                    raw_output = ""
                observation = JsonDataModel.from_raw_data(
                    {
                        "success": True,
                        "result": raw_output,
                        "images": results["result"]["outputs"] if ".show()" in results[
                            "intermediate_steps"] else [],
                        "intermediate_steps": results["intermediate_steps"],
                    },
                    filter_keys=["images"],
                )
            else:
                observation = JsonDataModel.from_raw_data(
                    {
                        "success": False,
                        "result": results["result"]["error_message"],
                        "intermediate_steps": results["intermediate_steps"],
                    }
                )
            return observation
        except Exception as e:
            logger.bind(msg_head=f"PythonCodeBuilder error({llm})").error(str(e))

            traceback.print_exc()
            results = basic_chat_executor.run(user_intent=term, llm=llm)
            return results["result"]

    def run_sql_code_builder(term: str) -> Union[Dict, DataModel]:
        try:

            def convert_grounding_source_as_db(
                    grounding_source_dict: Dict[str, DataModel]
            ) -> Union[List[TableDataModel], DatabaseDataModel]:
                db_grounding_source = [
                    gs for _, gs in grounding_source_dict.items() if
                    isinstance(gs, DatabaseDataModel)
                ]
                table_grounding_source = [
                    gs for _, gs in grounding_source_dict.items() if
                    isinstance(gs, TableDataModel)
                ]
                assert len(db_grounding_source) <= 1
                if len(table_grounding_source) == 0:
                    # Only DatabaseDataModel. Assume there is at least one grounding
                    # source
                    return db_grounding_source[0]


                        "intermediate_steps": results["intermediate_steps"],
                    },
                    filter_keys=["images"],
                )
            else:
                observation = JsonDataModel.from_raw_data(
                    {
                        "success": False,
                        "result": results["result"]["error_message"],
                        "intermediate_steps": results["intermediate_steps"],
                    }
                )
            return observation
        except Exception as e:
            logger.bind(msg_head=f"PythonCodeBuilder error({llm})").error(str(e))

            traceback.print_exc()
            results = basic_chat_executor.run(user_intent=term, llm=llm)
            return results["result"]

    def run_sql_code_builder(term: str) -> Union[Dict, DataModel]:
        try:

            def convert_grounding_source_as_db(
                    grounding_source_dict: Dict[str, DataModel]
            ) -> Union[List[TableDataModel], DatabaseDataModel]:
                db_grounding_source = [
                    gs for _, gs in grounding_source_dict.items() if
                    isinstance(gs, DatabaseDataModel)
                ]
                table_grounding_source = [
                    gs for _, gs in grounding_source_dict.items() if
                    isinstance(gs, TableDataModel)
                ]
                assert len(db_grounding_source) <= 1
                if len(table_grounding_source) == 0:
                    # Only DatabaseDataModel. Assume there is at least one grounding
                    # source
                    return db_grounding_source[0]
                else:
                    for t_gs in table_grounding_source:
                        if len(db_grounding_source) == 0:
                            # No DatabaseDataModel, then convert the first TableModel
                            # into DatabaseDataModel.
                            if t_gs.db_view is None:
                                t_gs.set_db_view(
                                    DatabaseDataModel.from_table_data_model(t_gs))
                            db_gs = t_gs.db_view
                            db_grounding_source.append(db_gs)
                        else:
                            # Insert TableDataModel into the existing DatabaseDataModel
                            db_gs = db_grounding_source[0]
                            db_gs.insert_table_data_model(t_gs)
                    return db_gs

            input_grounding_source = convert_grounding_source_as_db(
                grounding_source_dict)
            results = sql_code_generation_executor.run(
                user_intent=term,
                grounding_source=input_grounding_source,
                llm=llm,
            )

            logger.bind(msg_head=f"SQLQueryBuilder results({llm})").debug(results)

            if results["result"]["success"]:
                observation = JsonDataModel.from_raw_data({
                    "success": True,
                    "result": results["result"]["result"],
                    "intermediate_steps": results["intermediate_steps"],
                })
            else:
                observation = JsonDataModel.from_raw_data({
                    "success": False,
                    "result": results["result"]["error_message"],
                    "intermediate_steps": results["intermediate_steps"],
                })
            return observation
        except Exception as e:
            logger.bind(msg_head=f"SQLQueryBuilder results({llm})").error(str(e))

            traceback.print_exc()
            results = basic_chat_executor.run(user_intent=term, llm=llm)
            return results["result"]

    def run_echarts_interactive_plotter(term: str) -> Union[Dict, DataModel]:
        try:
            input_grounding_source = [gs for _, gs in grounding_source_dict.items() if
                                      isinstance(gs, TableDataModel)]
            results = echart_code_generation_executor.run(
                user_intent=term,
                llm=llm,
                grounding_source=input_grounding_source,
                user_id=user_id,
                chat_id=chat_id,
                code_execution_mode=code_execution_mode,
                jupyter_kernel_pool=jupyter_kernel_pool,
            )

            logger.bind(msg_head=f"PlotInteractivePlotter results({llm})").debug(
                results)

            if results["result"]["success"]:
                results = JsonDataModel.from_raw_data(


            if results["result"]["success"]:
                observation = JsonDataModel.from_raw_data({
                    "success": True,
                    "result": results["result"]["result"],
                    "intermediate_steps": results["intermediate_steps"],
                })
            else:
                observation = JsonDataModel.from_raw_data({
                    "success": False,
                    "result": results["result"]["error_message"],
                    "intermediate_steps": results["intermediate_steps"],
                })
            return observation
        except Exception as e:
            logger.bind(msg_head=f"SQLQueryBuilder results({llm})").error(str(e))

            traceback.print_exc()
            results = basic_chat_executor.run(user_intent=term, llm=llm)
            return results["result"]

    def run_echarts_interactive_plotter(term: str) -> Union[Dict, DataModel]:
        try:
            input_grounding_source = [gs for _, gs in grounding_source_dict.items() if
                                      isinstance(gs, TableDataModel)]
            results = echart_code_generation_executor.run(
                user_intent=term,
                llm=llm,
                grounding_source=input_grounding_source,
                user_id=user_id,
                chat_id=chat_id,
                code_execution_mode=code_execution_mode,
                jupyter_kernel_pool=jupyter_kernel_pool,
            )

            logger.bind(msg_head=f"PlotInteractivePlotter results({llm})").debug(
                results)

            if results["result"]["success"]:
                results = JsonDataModel.from_raw_data(
                    {
                        "success": True,
                        "result": "",
                        "echarts": polish_echarts(results["result"]["stdout"]),
                        "intermediate_steps": results["intermediate_steps"],
                    },
                    filter_keys=["result", "echarts"],
                )
            else:
                results = JsonDataModel.from_raw_data(
                    {
                        "success": False,
                        "result": results["result"]["error_message"],
                        "intermediate_steps": results["intermediate_steps"],
                    }
                )
            return results
        except Exception as e:
            logger.bind(msg_head=f"PlotInteractivePlotter error({llm})").error(str(e))

            results = basic_chat_executor.run(user_intent=term, llm=llm)
            return results["result"]

    def run_kaggle_data_loader(term: str) -> Union[Dict, DataModel]:
        try:
            results = kaggle_data_loading_executor.run(
                user_intent=term,
                llm=llm,
            )
            logger.bind(msg_head=f"KaggleDataLoader results({llm})").debug(results)

            results = JsonDataModel.from_raw_data(
                {
                    "success": True,
                    "kaggle_action": results["kaggle_action"],
                    "kaggle_output_info": results["kaggle_output_info"],
                },
            )
            return results
        except Exception as e:
            logger.bind(msg_head=f"KaggleDataLoader results({llm})").error(str(e))


                        "echarts": polish_echarts(results["result"]["stdout"]),
                        "intermediate_steps": results["intermediate_steps"],
                    },
                    filter_keys=["result", "echarts"],
                )
            else:
                results = JsonDataModel.from_raw_data(
                    {
                        "success": False,
                        "result": results["result"]["error_message"],
                        "intermediate_steps": results["intermediate_steps"],
                    }
                )
            return results
        except Exception as e:
            logger.bind(msg_head=f"PlotInteractivePlotter error({llm})").error(str(e))

            results = basic_chat_executor.run(user_intent=term, llm=llm)
            return results["result"]

    def run_kaggle_data_loader(term: str) -> Union[Dict, DataModel]:
        try:
            results = kaggle_data_loading_executor.run(
                user_intent=term,
                llm=llm,
            )
            logger.bind(msg_head=f"KaggleDataLoader results({llm})").debug(results)

            results = JsonDataModel.from_raw_data(
                {
                    "success": True,
                    "kaggle_action": results["kaggle_action"],
                    "kaggle_output_info": results["kaggle_output_info"],
                },
            )
            return results
        except Exception as e:
            logger.bind(msg_head=f"KaggleDataLoader results({llm})").error(str(e))

            traceback.print_exc()
            results = basic_chat_executor.run(user_intent=term, llm=llm)
            return results["result"]

    tool_dict = {
        "PythonCodeBuilder": Tool(
            name="PythonCodeBuilder",
            func=run_python_code_builder,
            description="""
Description: This tool adeptly turns your textual problem or query into Python code & execute it to get results. It shines when dealing with mathematics, data manipulation tasks, general computational problems and basic visualization like matplotlib. Please note it does not generate database queries.
Input: A natural language problem or question.
Output: A Python program + its execution result to solve the presented problem or answer the question.
Note: The tool MUST be used whenever you want to generate & execute Python code.
                """,
        ),
        "SQLQueryBuilder": Tool(
            name="SQLQueryBuilder",
            func=run_sql_code_builder,
            description="""
Description: Specialized for database tasks, this tool converts your natural language query into SQL code & execute it to get results. It's particularly suited for creating database queries, but it doesn't solve mathematical problems or perform data manipulations outside the SQL context. Be sure to specify the table name for successful operation.
Input: A natural language query related to database operations, along with the name of the table on which the query will operate.
Output: A SQL program, ready to execute on the specified database table, and its execution result.
Note: It is ALWAYS preferable to use the tool whenever you want to generate SQL query & execute the SQL query.


                )
            else:
                observation = JsonDataModel.from_raw_data(
                    {
                        "success": False,
                        "result": results["result"]["error_message"],
                        "intermediate_steps": results["intermediate_steps"],
                    }
                )
            return observation
        except Exception as e:
            logger.bind(msg_head=f"PythonCodeBuilder error({llm})").error(str(e))

            traceback.print_exc()
            results = basic_chat_executor.run(user_intent=term, llm=llm)
            return results["result"]

    def run_sql_code_builder(term: str) -> Union[Dict, DataModel]:
        try:

            def convert_grounding_source_as_db(
                    grounding_source_dict: Dict[str, DataModel]
            ) -> Union[List[TableDataModel], DatabaseDataModel]:
                db_grounding_source = [
                    gs for _, gs in grounding_source_dict.items() if
                    isinstance(gs, DatabaseDataModel)
                ]
                table_grounding_source = [
                    gs for _, gs in grounding_source_dict.items() if
                    isinstance(gs, TableDataModel)
                ]
                assert len(db_grounding_source) <= 1
                if len(table_grounding_source) == 0:
                    # Only DatabaseDataModel. Assume there is at least one grounding
                    # source
                    return db_grounding_source[0]
                else:
                    for t_gs in table_grounding_source:
                        if len(db_grounding_source) == 0:
                            # No DatabaseDataModel, then convert the first TableModel
                            # into DatabaseDataModel.
                            if t_gs.db_view is None:
                                t_gs.set_db_view(
                                    DatabaseDataModel.from_table_data_model(t_gs))
                            db_gs = t_gs.db_view
                            db_grounding_source.append(db_gs)
                        else:
                            # Insert TableDataModel into the existing DatabaseDataModel
                            db_gs = db_grounding_source[0]
                            db_gs.insert_table_data_model(t_gs)
                    return db_gs

            input_grounding_source = convert_grounding_source_as_db(
                grounding_source_dict)
            results = sql_code_generation_executor.run(
                user_intent=term,
                grounding_source=input_grounding_source,
                llm=llm,
            )

            logger.bind(msg_head=f"SQLQueryBuilder results({llm})").debug(results)

            if results["result"]["success"]:
                observation = JsonDataModel.from_raw_data({
                    "success": True,
                    "result": results["result"]["result"],
                    "intermediate_steps": results["intermediate_steps"],
                })
            else:
                observation = JsonDataModel.from_raw_data({
                    "success": False,


from flask import request, jsonify, Response

from backend.api.chat_webot import get_webot_from_redis, save_webot_to_redis
from backend.main import app
from backend.schemas import DEFAULT_USER_ID
from backend.api.language_model import get_llm

@app.route("/api/webot/action", methods=["POST"])
def get_action() -> Response:
    """Gets the next action to take for a given the current page HTML."""
    request_json = request.get_json()
    user_id = request_json.pop("user_id", DEFAULT_USER_ID)
    chat_id = request_json["chat_id"]
    webot = get_webot_from_redis(user_id=user_id, chat_id=chat_id)
    # Get request parameters
    request_json = request.get_json()
    processed_html = request_json["processed_html"]
    llm = get_llm("gpt-4")
    result = webot.run(processed_html, llm=llm)
    save_webot_to_redis(user_id=user_id, chat_id=chat_id, webot=webot)

    return jsonify({
        "chat_id": chat_id,
        "user_id": user_id,
        "action_response": result,
    })

@app.route("/api/webot/interrupt", methods=["POST"])
def interrupt() -> Response:
    """Interrupts the current webot."""
    request_json = request.get_json()
    user_id = request_json.pop("user_id", DEFAULT_USER_ID)
    chat_id = request_json["chat_id"]
    interrupt = request_json["interrupt"]
    if interrupt:
        webot = get_webot_from_redis(user_id=user_id, chat_id=chat_id)
        webot.actions_taken.append("interrupt")
        save_webot_to_redis(user_id=user_id, chat_id=chat_id, webot=webot)
        return jsonify({
            "chat_id": chat_id,
            "user_id": user_id,
        })
    return jsonify({"message": "No interrupt signal received."})

@app.route("/api/webot/error", methods=["POST"])
def error() -> Response:
    """Appends action 'error' to the current webot."""
    request_json = request.get_json()
    user_id = request_json.pop("user_id", DEFAULT_USER_ID)
    chat_id = request_json["chat_id"]
    error = request_json["error"]
    if error:
        webot = get_webot_from_redis(user_id=user_id, chat_id=chat_id)
        webot.actions_taken.append("error")
        save_webot_to_redis(user_id=user_id, chat_id=chat_id, webot=webot)
        return jsonify({
            "chat_id": chat_id,
            "user_id": user_id,
        })
    return jsonify({"message": "No error signal received."})


from flask import request, jsonify, Response

from backend.api.chat_webot import get_webot_from_redis, save_webot_to_redis
from backend.main import app
from backend.schemas import DEFAULT_USER_ID
from backend.api.language_model import get_llm

@app.route("/api/webot/action", methods=["POST"])
def get_action() -> Response:
    """Gets the next action to take for a given the current page HTML."""
    request_json = request.get_json()
    user_id = request_json.pop("user_id", DEFAULT_USER_ID)
    chat_id = request_json["chat_id"]
    webot = get_webot_from_redis(user_id=user_id, chat_id=chat_id)
    # Get request parameters
    request_json = request.get_json()
    processed_html = request_json["processed_html"]
    llm = get_llm("gpt-4")
    result = webot.run(processed_html, llm=llm)
    save_webot_to_redis(user_id=user_id, chat_id=chat_id, webot=webot)

    return jsonify({
        "chat_id": chat_id,
        "user_id": user_id,
        "action_response": result,
    })

@app.route("/api/webot/interrupt", methods=["POST"])
def interrupt() -> Response:
    """Interrupts the current webot."""
    request_json = request.get_json()
    user_id = request_json.pop("user_id", DEFAULT_USER_ID)
    chat_id = request_json["chat_id"]
    interrupt = request_json["interrupt"]
    if interrupt:
        webot = get_webot_from_redis(user_id=user_id, chat_id=chat_id)
        webot.actions_taken.append("interrupt")
        save_webot_to_redis(user_id=user_id, chat_id=chat_id, webot=webot)
        return jsonify({
            "chat_id": chat_id,
            "user_id": user_id,
        })
    return jsonify({"message": "No interrupt signal received."})


interrupt

    """Gets the next action to take for a given the current page HTML."""
    request_json = request.get_json()
    user_id = request_json.pop("user_id", DEFAULT_USER_ID)
    chat_id = request_json["chat_id"]
    webot = get_webot_from_redis(user_id=user_id, chat_id=chat_id)
    # Get request parameters
    request_json = request.get_json()
    processed_html = request_json["processed_html"]
    llm = get_llm("gpt-4")
    result = webot.run(processed_html, llm=llm)
    save_webot_to_redis(user_id=user_id, chat_id=chat_id, webot=webot)

    return jsonify({
        "chat_id": chat_id,
        "user_id": user_id,
        "action_response": result,
    })

@app.route("/api/webot/interrupt", methods=["POST"])
def interrupt() -> Response:
    """Interrupts the current webot."""
    request_json = request.get_json()
    user_id = request_json.pop("user_id", DEFAULT_USER_ID)
    chat_id = request_json["chat_id"]
    interrupt = request_json["interrupt"]
    if interrupt:
        webot = get_webot_from_redis(user_id=user_id, chat_id=chat_id)
        webot.actions_taken.append("interrupt")
        save_webot_to_redis(user_id=user_id, chat_id=chat_id, webot=webot)
        return jsonify({
            "chat_id": chat_id,
            "user_id": user_id,
        })
    return jsonify({"message": "No interrupt signal received."})

@app.route("/api/webot/error", methods=["POST"])
def error() -> Response:
    """Appends action 'error' to the current webot."""
    request_json = request.get_json()
    user_id = request_json.pop("user_id", DEFAULT_USER_ID)
    chat_id = request_json["chat_id"]
    error = request_json["error"]
    if error:
        webot = get_webot_from_redis(user_id=user_id, chat_id=chat_id)
        webot.actions_taken.append("error")
        save_webot_to_redis(user_id=user_id, chat_id=chat_id, webot=webot)
        return jsonify({
            "chat_id": chat_id,
            "user_id": user_id,
        })
    return jsonify({"message": "No error signal received."})


error

@app.route("/api/webot/interrupt", methods=["POST"])
def interrupt() -> Response:
    """Interrupts the current webot."""
    request_json = request.get_json()
    user_id = request_json.pop("user_id", DEFAULT_USER_ID)
    chat_id = request_json["chat_id"]
    interrupt = request_json["interrupt"]
    if interrupt:
        webot = get_webot_from_redis(user_id=user_id, chat_id=chat_id)
        webot.actions_taken.append("interrupt")
        save_webot_to_redis(user_id=user_id, chat_id=chat_id, webot=webot)
        return jsonify({
            "chat_id": chat_id,
            "user_id": user_id,
        })
    return jsonify({"message": "No interrupt signal received."})

@app.route("/api/webot/error", methods=["POST"])
def error() -> Response:
    """Appends action 'error' to the current webot."""
    request_json = request.get_json()
    user_id = request_json.pop("user_id", DEFAULT_USER_ID)
    chat_id = request_json["chat_id"]
    error = request_json["error"]
    if error:
        webot = get_webot_from_redis(user_id=user_id, chat_id=chat_id)
        webot.actions_taken.append("error")
        save_webot_to_redis(user_id=user_id, chat_id=chat_id, webot=webot)
        return jsonify({
            "chat_id": chat_id,
            "user_id": user_id,
        })
    return jsonify({"message": "No error signal received."})


import os

from backend.app import app
from real_agents.adapters.models import ChatOpenAI, ChatAnthropic
from real_agents.adapters.llm import BaseLanguageModel

LLAMA_DIR = "PATH_TO_LLAMA_DIR"

@app.route("/api/llm_list", methods=["POST"])
def get_llm_list():
    """Gets the whole llm list."""
    return [
        {"id": llm, "name": llm} for llm in [
            "gpt-3.5-turbo-16k",
            "gpt-4",
            "claude-v1",
            "claude-2",
            "lemur-chat"
        ]
    ]

def get_llm(llm_name: str, **kwargs) -> BaseLanguageModel:
    """Gets the llm model by its name."""
    if llm_name in ["gpt-3.5-turbo-16k", "gpt-4"]:
        return ChatOpenAI(
            model_name=llm_name,
            streaming=True,
            verbose=True,
            **kwargs
        )
    elif llm_name in ["claude-v1", "claude-2"]:
        anthropic_api_key = os.getenv("ANTHROPIC_API_KEY", "")
        return ChatAnthropic(
            model=llm_name,
            streaming=True,
            verbose=True,
            anthropic_api_key=anthropic_api_key,
            **kwargs,
        )
    elif llm_name == "lemur-chat":
        return ChatOpenAI(
            model_name="lemur-70b-chat-v1",
            streaming=True,
            openai_api_base="https://model-api.xlang.ai/v1",
            verbose=True,
            max_tokens=2048,
            **kwargs
        )
    else:
        raise ValueError(f"llm_name {llm_name} not found")


import os

from backend.app import app
from real_agents.adapters.models import ChatOpenAI, ChatAnthropic
from real_agents.adapters.llm import BaseLanguageModel

LLAMA_DIR = "PATH_TO_LLAMA_DIR"

@app.route("/api/llm_list", methods=["POST"])
def get_llm_list():
    """Gets the whole llm list."""
    return [
        {"id": llm, "name": llm} for llm in [
            "gpt-3.5-turbo-16k",
            "gpt-4",
            "claude-v1",
            "claude-2",
            "lemur-chat"
        ]
    ]

def get_llm(llm_name: str, **kwargs) -> BaseLanguageModel:
    """Gets the llm model by its name."""
    if llm_name in ["gpt-3.5-turbo-16k", "gpt-4"]:
        return ChatOpenAI(
            model_name=llm_name,
            streaming=True,
            verbose=True,
            **kwargs
        )
    elif llm_name in ["claude-v1", "claude-2"]:
        anthropic_api_key = os.getenv("ANTHROPIC_API_KEY", "")
        return ChatAnthropic(
            model=llm_name,
            streaming=True,
            verbose=True,
            anthropic_api_key=anthropic_api_key,
            **kwargs,
        )


from real_agents.adapters.models import ChatOpenAI, ChatAnthropic
from real_agents.adapters.llm import BaseLanguageModel

LLAMA_DIR = "PATH_TO_LLAMA_DIR"

@app.route("/api/llm_list", methods=["POST"])
def get_llm_list():
    """Gets the whole llm list."""
    return [
        {"id": llm, "name": llm} for llm in [
            "gpt-3.5-turbo-16k",
            "gpt-4",
            "claude-v1",
            "claude-2",
            "lemur-chat"
        ]
    ]

def get_llm(llm_name: str, **kwargs) -> BaseLanguageModel:
    """Gets the llm model by its name."""
    if llm_name in ["gpt-3.5-turbo-16k", "gpt-4"]:
        return ChatOpenAI(
            model_name=llm_name,
            streaming=True,
            verbose=True,
            **kwargs
        )
    elif llm_name in ["claude-v1", "claude-2"]:
        anthropic_api_key = os.getenv("ANTHROPIC_API_KEY", "")
        return ChatAnthropic(
            model=llm_name,
            streaming=True,
            verbose=True,
            anthropic_api_key=anthropic_api_key,
            **kwargs,
        )
    elif llm_name == "lemur-chat":
        return ChatOpenAI(
            model_name="lemur-70b-chat-v1",
            streaming=True,
            openai_api_base="https://model-api.xlang.ai/v1",
            verbose=True,
            max_tokens=2048,
            **kwargs
        )
    else:
        raise ValueError(f"llm_name {llm_name} not found")


from flask import request, jsonify, Response

from backend.main import app
from backend.schemas import DEFAULT_USER_ID
from backend.api.chat_webot import get_webot_from_redis, \
    get_webot_status_from_redis, reset_webot_status

@app.route("/api/webot/instructions", methods=["POST"])
def get_instruction() -> Response:
    request_json = request.get_json()
    user_id = request_json.pop("user_id", DEFAULT_USER_ID)
    chat_id = request_json["chat_id"]
    webot = get_webot_from_redis(user_id=user_id, chat_id=chat_id)
    return jsonify({
        "chat_id": chat_id,
        "user_id": user_id,
        "instructions": webot.instruction
    })

@app.route("/api/webot/webot_status", methods=["POST"])
def get_webot_status() -> Response:
    request_json = request.get_json()
    user_id = request_json.pop("user_id", DEFAULT_USER_ID)
    chat_id = request_json["chat_id"]
    webot_status_json = get_webot_status_from_redis(user_id=user_id, chat_id=chat_id)
    return jsonify(webot_status_json) if webot_status_json is not None else jsonify(
        {"webot_status": None, "url": None})

@app.route("/api/webot/reset_status", methods=["POST"])
def reset_status() -> Response:
    request_json = request.get_json()
    user_id = request_json.pop("user_id", DEFAULT_USER_ID)
    chat_id = request_json["chat_id"]
    reset_webot_status(user_id=user_id, chat_id=chat_id)
    return jsonify({
        "chat_id": chat_id,
        "user_id": user_id,
    })


from flask import request, jsonify, Response

from backend.main import app
from backend.schemas import DEFAULT_USER_ID
from backend.api.chat_webot import get_webot_from_redis, \
    get_webot_status_from_redis, reset_webot_status

@app.route("/api/webot/instructions", methods=["POST"])
def get_instruction() -> Response:
    request_json = request.get_json()
    user_id = request_json.pop("user_id", DEFAULT_USER_ID)
    chat_id = request_json["chat_id"]
    webot = get_webot_from_redis(user_id=user_id, chat_id=chat_id)
    return jsonify({
        "chat_id": chat_id,
        "user_id": user_id,
        "instructions": webot.instruction
    })

@app.route("/api/webot/webot_status", methods=["POST"])
def get_webot_status() -> Response:
    request_json = request.get_json()
    user_id = request_json.pop("user_id", DEFAULT_USER_ID)
    chat_id = request_json["chat_id"]
    webot_status_json = get_webot_status_from_redis(user_id=user_id, chat_id=chat_id)
    return jsonify(webot_status_json) if webot_status_json is not None else jsonify(
        {"webot_status": None, "url": None})

@app.route("/api/webot/reset_status", methods=["POST"])
def reset_status() -> Response:
    request_json = request.get_json()
    user_id = request_json.pop("user_id", DEFAULT_USER_ID)
    chat_id = request_json["chat_id"]
    reset_webot_status(user_id=user_id, chat_id=chat_id)
    return jsonify({
        "chat_id": chat_id,


from backend.main import app
from backend.schemas import DEFAULT_USER_ID
from backend.api.chat_webot import get_webot_from_redis, \
    get_webot_status_from_redis, reset_webot_status

@app.route("/api/webot/instructions", methods=["POST"])
def get_instruction() -> Response:
    request_json = request.get_json()
    user_id = request_json.pop("user_id", DEFAULT_USER_ID)
    chat_id = request_json["chat_id"]
    webot = get_webot_from_redis(user_id=user_id, chat_id=chat_id)
    return jsonify({
        "chat_id": chat_id,
        "user_id": user_id,
        "instructions": webot.instruction
    })

@app.route("/api/webot/webot_status", methods=["POST"])
def get_webot_status() -> Response:
    request_json = request.get_json()
    user_id = request_json.pop("user_id", DEFAULT_USER_ID)
    chat_id = request_json["chat_id"]
    webot_status_json = get_webot_status_from_redis(user_id=user_id, chat_id=chat_id)
    return jsonify(webot_status_json) if webot_status_json is not None else jsonify(
        {"webot_status": None, "url": None})

@app.route("/api/webot/reset_status", methods=["POST"])
def reset_status() -> Response:
    request_json = request.get_json()
    user_id = request_json.pop("user_id", DEFAULT_USER_ID)
    chat_id = request_json["chat_id"]
    reset_webot_status(user_id=user_id, chat_id=chat_id)
    return jsonify({
        "chat_id": chat_id,
        "user_id": user_id,
    })


    chat_id = request_json["chat_id"]
    webot = get_webot_from_redis(user_id=user_id, chat_id=chat_id)
    return jsonify({
        "chat_id": chat_id,
        "user_id": user_id,
        "instructions": webot.instruction
    })

@app.route("/api/webot/webot_status", methods=["POST"])
def get_webot_status() -> Response:
    request_json = request.get_json()
    user_id = request_json.pop("user_id", DEFAULT_USER_ID)
    chat_id = request_json["chat_id"]
    webot_status_json = get_webot_status_from_redis(user_id=user_id, chat_id=chat_id)
    return jsonify(webot_status_json) if webot_status_json is not None else jsonify(
        {"webot_status": None, "url": None})

@app.route("/api/webot/reset_status", methods=["POST"])
def reset_status() -> Response:
    request_json = request.get_json()
    user_id = request_json.pop("user_id", DEFAULT_USER_ID)
    chat_id = request_json["chat_id"]
    reset_webot_status(user_id=user_id, chat_id=chat_id)
    return jsonify({
        "chat_id": chat_id,
        "user_id": user_id,
    })


backend.api.recommend

from typing import Dict
from flask import request, jsonify, Response

from backend.main import message_pool
from backend.app import app
from backend.api.language_model import get_llm
from backend.utils.utils import get_user_and_chat_id_from_request_json
from real_agents.adapters.executors import QuestionSuggestionExecutor
from real_agents.adapters.memory import ConversationReActBufferMemory

@app.route("/api/recommend", methods=["POST"])
def recommend() -> dict | Response:
    """Recommends potential inputs for users. """
    try:
        request_json = request.get_json()
        (user_id, chat_id) = get_user_and_chat_id_from_request_json(request_json)
        parent_message_id = int(request_json["parent_message_id"])
        user_intent = request_json["user_intent"]

        # Find the mainstat message list from leaf to root
        activated_message_list = message_pool.get_activated_message_list(
            user_id, chat_id, default_value=list(), parent_message_id=parent_message_id
        )
        chat_memory = ConversationReActBufferMemory(memory_key="chat_history", return_messages=True)
        message_pool.load_agent_memory_from_list(chat_memory, activated_message_list)
        question_suggestion_executor = QuestionSuggestionExecutor()

        llm_name = request_json["llm_name"]
        temperature = request_json.get("temperature", 0.7)
        kwargs = {
            "temperature": temperature,
        }

        # Get language model
        llm = get_llm(llm_name, **kwargs)
        follow_questions = question_suggestion_executor.run(
            user_intent=user_intent,
            llm=llm,
            chat_memory=chat_memory,
            mode="chat_memory",
        )

        return jsonify({
            "recommend_questions": follow_questions["questions"],
            "user_id": user_id,
            "chat_id": chat_id,
        })
    except Exception as e:
        import traceback
        traceback.print_exc()
        return {
            "recommend_questions": [],
            "user_id": user_id,
            "chat_id": chat_id,
        }


from typing import List
from flask import jsonify

from backend.app import app

DATA_TOOLS = [
    {
        "type": "language",
        "id": "1cea1f39-fe63-4b08-83d5-fa4c93db0c87",
        "name": "SQLQueryBuilder",
        "name_for_human": "SQL",
        "pretty_name_for_human": "SQL Query Generation",
        "icon": "",
        "description": "Using SQL as the programming language",
    },
    {
        "type": "language",
        "id": "0c135359-af7e-473b-8425-1393d2943b57",
        "name": "PythonCodeBuilder",
        "name_for_human": "Python",
        "pretty_name_for_human": "Python Code Generation",
        "icon": "",
        "description": "Using Python as the programming language",
    },
    {
        "type": "tool",
        "id": "a86aebe1-a780-4038-a333-fb2a9d2d25fc",
        "name": "Echarts",
        "name_for_human": "Echarts",
        "pretty_name_for_human": "Echarts",
        "icon": "",
        "description": "Enhancing the analyzing experience with interactive charts",
    },
    {
        "type": "tool",
        "id": "c7c826ba-5884-4e2b-b27c-fedea30c1749",
        "name": "KaggleDataLoader",
        "name_for_human": "Kaggle Data Search",
        "pretty_name_for_human": "Kaggle Data Search",
        "icon": "",
        "description": "Search & Connect to kaggle datasets",
    },
    {
        "type": "tool",
        "id": "8f8e8dbc-ae5b-4950-9f4f-7f5238978806",
        "name": "DataProfiling",
        "name_for_human": "Data Profiling",
        "pretty_name_for_human": "Data Profiling",
        "icon": "",
        "description": "Intelligent profiling for your data",
    },
]

@app.route("/api/data_tool_list", methods=["POST"])
def get_data_tool_list() -> List[dict]:
    """Gets the data tool list. """
    for i, tool in enumerate(DATA_TOOLS):
        cache_path = f"backend/static/images/{tool['name']}.cache"
        with open(cache_path, 'r') as f:
            image_content = f.read()
            DATA_TOOLS[i]["icon"] = image_content

    return jsonify(DATA_TOOLS)


        "id": "c7c826ba-5884-4e2b-b27c-fedea30c1749",
        "name": "KaggleDataLoader",
        "name_for_human": "Kaggle Data Search",
        "pretty_name_for_human": "Kaggle Data Search",
        "icon": "",
        "description": "Search & Connect to kaggle datasets",
    },
    {
        "type": "tool",
        "id": "8f8e8dbc-ae5b-4950-9f4f-7f5238978806",
        "name": "DataProfiling",
        "name_for_human": "Data Profiling",
        "pretty_name_for_human": "Data Profiling",
        "icon": "",
        "description": "Intelligent profiling for your data",
    },
]

@app.route("/api/data_tool_list", methods=["POST"])
def get_data_tool_list() -> List[dict]:
    """Gets the data tool list. """
    for i, tool in enumerate(DATA_TOOLS):
        cache_path = f"backend/static/images/{tool['name']}.cache"
        with open(cache_path, 'r') as f:
            image_content = f.read()
            DATA_TOOLS[i]["icon"] = image_content

    return jsonify(DATA_TOOLS)


backend.api.tool

from flask import request, jsonify, Response

from backend.api.chat_plugin import plugins
from backend.main import app, api_key_pool
from backend.schemas import DEFAULT_USER_ID

@app.route("/api/tool_list", methods=["POST"])
def get_tool_list() -> Response:
    """parameters:
    {
      user_id: id of the user
    }
    return value:
    [{
        id: id of a plugin,
        name: name pf a plugin,
        description: description of the plugin,
        icon: icon of the plugin,
        require_api_key: whether the plugin requires api_key,
        api_key: the api key of the plugin, None if no api key
    }]
    """
    user_id = DEFAULT_USER_ID
    api_key_info = api_key_pool.get_pool_info_with_id(user_id, [])
    tool_list = []
    for plugin in plugins:
        plugin_info = {
            "id": plugin["id"],
            "name": plugin["name"],
            "name_for_human": plugin["name_for_human"],
            "description": plugin["description"],
            "icon": plugin["icon"],
            "require_api_key": plugin["require_api_key"],
        }
        search_plugin = [i for i in api_key_info if i["tool_id"] == plugin["id"]]
        if len(search_plugin) > 0:
            plugin_info["api_key"] = search_plugin[0]["api_key"]
        else:
            plugin_info["api_key"] = None
        tool_list.append(plugin_info)
    return jsonify(tool_list)

@app.route("/api/api_key", methods=["POST"])
def post_tool_api_key() -> Response:
    """parameters:
    {
      user_id: id of the user,
      tool_id: id of the tool,
      tool_name: name of the tool,
      api_key: api_key of the tool
    }
    """
    request_json = request.get_json()
    user_id = request_json.pop("user_id", DEFAULT_USER_ID)
    tool_id = request_json["tool_id"]
    tool_name = request_json["tool_name"]
    api_key = request_json["api_key"]
    api_key_info = api_key_pool.get_pool_info_with_id(user_id, [])
    flag = False
    for i in api_key_info:
        if i["tool_id"] == tool_id:
            flag = True
            i["api_key"] = api_key
    if not flag:
        api_key_info.append({"tool_id": tool_id, "tool_name": tool_name, "api_key": api_key})
    api_key_pool.set_pool_info_with_id(user_id, api_key_info)
    return Response("Success", status=200)


from flask import request, jsonify, Response

from backend.api.chat_plugin import plugins
from backend.main import app, api_key_pool
from backend.schemas import DEFAULT_USER_ID

@app.route("/api/tool_list", methods=["POST"])
def get_tool_list() -> Response:
    """parameters:
    {
      user_id: id of the user
    }
    return value:
    [{
        id: id of a plugin,
        name: name pf a plugin,
        description: description of the plugin,
        icon: icon of the plugin,
        require_api_key: whether the plugin requires api_key,
        api_key: the api key of the plugin, None if no api key
    }]
    """
    user_id = DEFAULT_USER_ID
    api_key_info = api_key_pool.get_pool_info_with_id(user_id, [])
    tool_list = []
    for plugin in plugins:
        plugin_info = {
            "id": plugin["id"],
            "name": plugin["name"],
            "name_for_human": plugin["name_for_human"],
            "description": plugin["description"],
            "icon": plugin["icon"],
            "require_api_key": plugin["require_api_key"],
        }
        search_plugin = [i for i in api_key_info if i["tool_id"] == plugin["id"]]
        if len(search_plugin) > 0:
            plugin_info["api_key"] = search_plugin[0]["api_key"]
        else:
            plugin_info["api_key"] = None
        tool_list.append(plugin_info)
    return jsonify(tool_list)

@app.route("/api/api_key", methods=["POST"])
def post_tool_api_key() -> Response:
    """parameters:
    {
      user_id: id of the user,
      tool_id: id of the tool,
      tool_name: name of the tool,
      api_key: api_key of the tool
    }
    """
    request_json = request.get_json()
    user_id = request_json.pop("user_id", DEFAULT_USER_ID)
    tool_id = request_json["tool_id"]
    tool_name = request_json["tool_name"]
    api_key = request_json["api_key"]
    api_key_info = api_key_pool.get_pool_info_with_id(user_id, [])
    flag = False
    for i in api_key_info:


    tool_list = []
    for plugin in plugins:
        plugin_info = {
            "id": plugin["id"],
            "name": plugin["name"],
            "name_for_human": plugin["name_for_human"],
            "description": plugin["description"],
            "icon": plugin["icon"],
            "require_api_key": plugin["require_api_key"],
        }
        search_plugin = [i for i in api_key_info if i["tool_id"] == plugin["id"]]
        if len(search_plugin) > 0:
            plugin_info["api_key"] = search_plugin[0]["api_key"]
        else:
            plugin_info["api_key"] = None
        tool_list.append(plugin_info)
    return jsonify(tool_list)

@app.route("/api/api_key", methods=["POST"])
def post_tool_api_key() -> Response:
    """parameters:
    {
      user_id: id of the user,
      tool_id: id of the tool,
      tool_name: name of the tool,
      api_key: api_key of the tool
    }
    """
    request_json = request.get_json()
    user_id = request_json.pop("user_id", DEFAULT_USER_ID)
    tool_id = request_json["tool_id"]
    tool_name = request_json["tool_name"]
    api_key = request_json["api_key"]
    api_key_info = api_key_pool.get_pool_info_with_id(user_id, [])
    flag = False
    for i in api_key_info:
        if i["tool_id"] == tool_id:
            flag = True
            i["api_key"] = api_key
    if not flag:
        api_key_info.append({"tool_id": tool_id, "tool_name": tool_name, "api_key": api_key})
    api_key_pool.set_pool_info_with_id(user_id, api_key_info)
    return Response("Success", status=200)


backend.api.file

import json
import os
import shutil
from typing import Dict, Any
from flask import Response, jsonify, request, send_file

from backend.app import app
from backend.main import (
    grounding_source_pool,
    logger,
    message_id_register,
    message_pool,
)
from backend.schemas import DEFAULT_USER_ID
from backend.utils.utils import create_personal_folder
from backend.utils.user_conversation_storage import get_user_conversation_storage
from backend.utils.utils import (
    allowed_file,
    get_data_model_cls,
    get_user_and_chat_id_from_request,
    get_user_and_chat_id_from_request_json,
    is_sqlite_file,
    is_table_file,
    is_image_file,
    load_grounding_source,
)
from backend.schemas import INTERNAL, UNFOUND

TABLE_HUMAN_SIDE_FORMAT = "material-react-table"

def _path_tree_for_react_dnd_treeview(tree: list, id_to_path_dict: dict, path: str,
                                      parent: int,
                                      highlighted_files: list = []) -> list:
    """
    {
        "id": 1,
        "parent": 0,
        "droppable": true,
        "text": "Folder 1"
    },
    {
        "id": 2,
        "parent": 1,
        "text": "File 1-1",
        "data": {
            "fileType": "csv",
            "fileSize": "0.5MB"
        }
    },
    """
    for item in os.listdir(path):
        if item.startswith("."):
            continue
        item_path = os.path.join(path, item)
        droppable = os.path.isdir(item_path)
        idx = len(tree) + 1
        tree.append({
            "id": idx,
            "parent": parent,
            "droppable": droppable,
            "text": item,
            "highlight": True if item_path in highlighted_files else False})
        id_to_path_dict[idx] = item_path
        if os.path.isdir(item_path):
            _path_tree_for_react_dnd_treeview(tree, id_to_path_dict, item_path, idx)

    return []

def secure_filename(filename: str) -> str:
    keep_characters = ('.', '_')
    filename = "".join(
        c for c in filename if c.isalnum() or c in keep_characters).rstrip()
    return filename

@app.route("/api/upload", methods=["POST"])
def create_upload_file() -> dict | Response:
    """Uploads a new file."""
    try:
        if "file" not in request.files:
            return {"error": "No file part in the request"}
        file = request.files["file"]
        (user_id, chat_id) = get_user_and_chat_id_from_request(request)
        folder = create_personal_folder(user_id)

        # Check if the file is allowed
        if not allowed_file(str(file.filename)):
            return {"error": "File type not allowed"}

        # Save and read the file
        file.filename = secure_filename(str(file.filename))
        file_path = os.path.join(folder, file.filename)
        file.save(file_path)
        response = {"success": file.filename}

        logger.bind(user_id=user_id, chat_id=chat_id, api="/upload",
                    msg_head="Upload file success").debug(file_path)

        return jsonify(response)
    except Exception as e:
        logger.bind(user_id=user_id, chat_id=chat_id, api="/upload",
                    msg_head="Upload file error").error(str(e))

        return Response(response=None, status=f"{INTERNAL} Upload File Error: {str(e)}")

def _generate_human_side_data_from_file(filename: str, data_model: Any) -> Dict:
    if is_table_file(filename):
        # Determine the format of the human side(frontend) table
        human_side_data = data_model.get_human_side_data(mode="FULL")
        if TABLE_HUMAN_SIDE_FORMAT == "markdown":
            human_side_data = human_side_data.to_markdown(index=False)
            human_side_data_type = "plain"
        elif TABLE_HUMAN_SIDE_FORMAT == "material-react-table":
            columns = list(map(lambda item: {"accessorKey": item, "header": item},
                               human_side_data.columns.tolist()))
            data = human_side_data.fillna("").to_dict(orient="records")
            human_side_data = json.dumps({"columns": columns, "data": data})
            human_side_data_type = "table"
        data = {"success": filename, "content": human_side_data,
                "type": human_side_data_type}
    elif is_sqlite_file(filename):
        data = {"success": filename, "content": filename, "type": "table"}
    elif is_image_file(filename):
        # Determine the format of human side(frontend) image
        human_side_data = data_model.get_human_side_data()
        data = {"success": filename, "content": human_side_data, "type": "image"}
    else:
        return {"error": "Document file type not supported"}
    return data

def _get_file_path_from_node(folder: str, file_node: dict) -> Any:
    path_tree_list: list = []
    id_to_path_dict = {0: folder}
    _path_tree_for_react_dnd_treeview(path_tree_list, id_to_path_dict, folder, 0)
    path = id_to_path_dict[file_node["id"]]
    return path

@app.route("/api/file_system/apply", methods=["POST"])
def apply_to_conversation() -> Response:
    """Applies data to the conversation."""
    try:
        request_json = request.get_json()
        (user_id, chat_id) = get_user_and_chat_id_from_request_json(request_json)
        file_node = request_json["activated_file"]
        parent_message_id = request_json["parent_message_id"]
        folder = create_personal_folder(user_id)

        # Modify the selected grounding sources
        grounding_source_dict = grounding_source_pool.get_pool_info_with_id(user_id,
                                                                            chat_id,
                                                                            default_value={})
        file_path = _get_file_path_from_node(folder, file_node)
        filename = file_node["text"]
        filename_no_ext = os.path.splitext(filename)[0]
        if file_path not in grounding_source_dict:
            data = load_grounding_source(file_path)
            data_model = get_data_model_cls(filename).from_raw_data(
                raw_data=data,
                raw_data_name=filename_no_ext,
                raw_data_path=file_path,
            )
            grounding_source_dict[file_path] = data_model
            # Add uploaded file in chat memory
            message_list = message_pool.get_pool_info_with_id(user_id, chat_id,
                                                              default_value=list())
            llm_side_data = data_model.get_llm_side_data()
            human_message_content = "[User uploaded a file {}]\n{}".format(filename,
                                                                           llm_side_data)
            human_message_id = message_id_register.add_variable(human_message_content)
            message_list.append(
                {
                    "message_id": human_message_id,
                    "parent_message_id": parent_message_id,
                    "message_type": "human_message",
                    "message_content": human_message_content,
                }
            )
            data = _generate_human_side_data_from_file(filename, data_model)
            message_pool.set_pool_info_with_id(user_id, chat_id, message_list)
            grounding_source_pool.set_pool_info_with_id(user_id, chat_id,
                                                        grounding_source_dict)
            # Dump to database
            db = get_user_conversation_storage()
            db_message = {
                "conversation_id": chat_id,
                "user_id": user_id,
                "message_id": human_message_id,
                "parent_message_id": parent_message_id,
                "version_id": 0,
                "role": "user",
                "data_for_human": {
                    "intermediate_steps": [],
                    "final_answer": [
                        {
                            "type": data["type"],
                            "text": data["content"],
                            "final": True,
                        }
                    ],
                },
                "data_for_llm": message_list[-1]["message_content"],
                "raw_data": None,
            }
            db.message.insert_one(db_message)
            response = {
                "success": True,
                "message_id": human_message_id,
                "parent_message_id": parent_message_id,
                "message": "Successfully apply {} to conversation {}".format(filename,
                                                                             chat_id),
                "content": {
                    "intermediate_steps": [],
                    "final_answer": [
                        {
                            "type": data["type"],
                            "text": data["content"],
                            "final": True,
                        }
                    ],
                },
            }

            logger.bind(user_id=user_id, chat_id=chat_id, api="/apply",
                        msg_head="Apply file success").debug(file_path)
            del db_message["data_for_human"]

            return jsonify(response)
        else:
            logger.bind(user_id=user_id, chat_id=chat_id, api="/apply",
                        msg_head="Apply file failed").debug(file_path)

            return jsonify({"success": False,
                            "message": "You have already import {} to the conversation".format(
                                filename)})
    except Exception as e:
        logger.bind(user_id=user_id, chat_id=chat_id, api="/apply",
                    msg_head="Apply file failed").error(file_path)
        import traceback
        traceback.print_exc()

        return Response(response=None,
                        status=f"{INTERNAL} Fail to apply file to chat: {str(e)}")

@app.route("/api/file_system/move", methods=["POST"])
def move_files() -> Response:
    """Moves file from source path from target source."""
    request_json = request.get_json()
    (user_id, chat_id) = get_user_and_chat_id_from_request_json(request_json)
    root_path = create_personal_folder(user_id)
    nodes = request_json["nodes"]
    try:
        if os.path.exists(root_path) and os.path.isdir(root_path):
            current_path_tree_list: list = []
            id_to_path_dict = {0: root_path}
            _path_tree_for_react_dnd_treeview(current_path_tree_list, id_to_path_dict,
                                              root_path, 0)
        for node in nodes:
            old_path = id_to_path_dict[node["id"]]
            new_path = id_to_path_dict[node["parent"]]
            shutil.move(old_path, new_path)

            logger.bind(user_id=user_id, chat_id=chat_id, api="/move",
                        msg_head="Move file success").debug(
                f"from {old_path} to {new_path}"
            )

            return jsonify({"success": True, "message": "File moved successfully"})
    except Exception as e:
        logger.bind(user_id=user_id, chat_id=chat_id, api="/move",
                    msg_head="Move file failed").error(str(e))

        return jsonify({"success": False, "message": str(e)})
    return Response(response=None, status=f"{INTERNAL} Fail to move file")

@app.route("/api/file_system/delete", methods=["POST"])
def delete_files() -> Response:
    """Deletes a file from the filesystem."""
    request_json = request.get_json()
    (user_id, chat_id) = get_user_and_chat_id_from_request_json(request_json)
    root_path = create_personal_folder(user_id)
    node = request_json["node"]
    try:
        if os.path.exists(root_path) and os.path.isdir(root_path):
            current_path_tree_list: list = []
            id_to_path_dict = {0: root_path}
            _path_tree_for_react_dnd_treeview(current_path_tree_list, id_to_path_dict,
                                              root_path, 0)
            path = id_to_path_dict[node["id"]]
            if os.path.isdir(path):
                shutil.rmtree(path)
            else:
                os.remove(path)

        logger.bind(user_id=user_id, chat_id=chat_id, api="/delete",
                    msg_head="Delete file success").debug(path)

        return jsonify({"success": True, "message": "File is deleted successfully"})
    except Exception as e:
        logger.bind(user_id=user_id, chat_id=chat_id, api="/delete",
                    msg_head="Delete file failed").error(str(e))

        return Response(response=None,
                        status=f"{INTERNAL} Delete file failed: {str(e)}")

@app.route("/api/file_system/download", methods=["POST"])
def download_files() -> Response:
    """Downloads a file to local."""
    request_json = request.get_json()
    user_id = request_json.pop("user_id", DEFAULT_USER_ID)
    root_path = create_personal_folder(user_id)
    node = request_json["node"]

    try:
        if os.path.exists(root_path) and os.path.isdir(root_path):
            current_path_tree_list: list = []
            id_to_path_dict = {0: root_path}
            _path_tree_for_react_dnd_treeview(current_path_tree_list, id_to_path_dict,
                                              root_path, 0)
            path = id_to_path_dict[node["id"]]

            if os.path.exists(path):
                logger.bind(user_id=user_id, api="/download",
                            msg_head="download file success").debug(path)
                return send_file(path, as_attachment=True)

        logger.bind(user_id=user_id, api="/download",
                    msg_head="download file failed").debug(path)
        return Response(response=None,
                        status=f"{INTERNAL} Download file failed: file not correctlt sent")

    except Exception as e:
        print(str(e))
        import traceback
        traceback.print_exc()

        logger.bind(user_id=user_id, api="/download",
                    msg_head="download file failed").error(str(e))

        return Response(response=None,
                        status=f"{INTERNAL} Download file failed: {str(e)}")

def _generate_directory_name(name: str, x:int=0) -> Any:
    dir_name = (name + ("_" + str(x) if x != 0 else "")).strip()
    if not os.path.exists(dir_name):
        return dir_name
    else:
        return _generate_directory_name(name, x + 1)

@app.route("/api/file_system/create_folder", methods=["POST"])
def create_folders() -> Response:
    """Creates a folder in the filesystem."""
    request_json = request.get_json()
    (user_id, chat_id) = get_user_and_chat_id_from_request_json(request_json)
    root_path = create_personal_folder(user_id)
    if os.path.exists(root_path) and os.path.isdir(root_path):
        try:
            new_path = _generate_directory_name(os.path.join(root_path, "Folder"))
            os.makedirs(new_path, exist_ok=False)

            logger.bind(
                user_id=user_id, chat_id=chat_id, api="/create_folder",
                msg_head="Create folder success"
            ).debug(new_path)

            return jsonify({"success": True, "message": "Folder created successfully"})
        except Exception as e:
            logger.bind(user_id=user_id, chat_id=chat_id, api="/create_folder",
                        msg_head="Create folder failed").error(
                str(e)
            )

            return jsonify({"success": False, "message": str(e)})
    else:
        logger.bind(user_id=user_id, chat_id=chat_id, api="/create_folder",
                    msg_head="Create folder failed").error(
            "Root path does not exist."
        )

        return Response(response=None, status=f"{INTERNAL} Root path does not exist")

@app.route("/api/file_system/update", methods=["POST"])
def rename_folder() -> Response:
    """Renames a folder in the filesystem."""
    request_json = request.get_json()
    (user_id, chat_id) = get_user_and_chat_id_from_request_json(request_json)
    root_path = create_personal_folder(user_id)
    node = request_json["node"]
    rename_value = request_json["rename_value"]
    if os.path.exists(root_path) and os.path.isdir(root_path):
        try:
            current_path_tree_list: list = []
            id_to_path_dict = {0: root_path}
            _path_tree_for_react_dnd_treeview(current_path_tree_list, id_to_path_dict,
                                              root_path, 0)
            path = id_to_path_dict[node["id"]]
            new_path = os.path.join(os.path.dirname(path), rename_value)
            shutil.move(path, new_path)

            logger.bind(user_id=user_id, chat_id=chat_id, api="/update",
                        msg_head="Rename folder success").debug(
                f"{path} to {new_path}"
            )

            return jsonify({"success": True, "message": "Folder created successfully"})
        except Exception as e:
            logger.bind(user_id=user_id, chat_id=chat_id, api="/update",
                        msg_head="Rename folder failed").error(str(e))

            return jsonify({"success": False, "message": str(e)})
    else:
        logger.bind(user_id=user_id, chat_id=chat_id, api="/update",
                    msg_head="Rename folder failed").error(
            "Root path does not exist."
        )

        return Response(response=None, status=f"{INTERNAL} Root path does not exist")

@app.route("/api/file_system/get_path_tree", methods=["POST"])
def get_path_tree() -> Response:
    """Gets a file path tree of one file."""
    try:
        request_json = request.get_json()
        user_id = request_json.pop("user_id", DEFAULT_USER_ID)
        if user_id == "":  # front-end may enter empty user_id
            return jsonify([])
        root_path = create_personal_folder(user_id)
        highlighted_files = request_json.get("highlighted_files", [])
        if root_path is None:
            return {"error": "root_path parameter is required", "error_code": 404}
        if os.path.exists(root_path) and os.path.isdir(root_path):
            current_path_tree_list: list = []
            id_to_path_dict = {0: root_path}
            _path_tree_for_react_dnd_treeview(current_path_tree_list, id_to_path_dict,
                                              root_path, 0,
                                              highlighted_files=highlighted_files)
            return jsonify(current_path_tree_list)
        else:
            return Response(response=None, status=f"{UNFOUND} Directory not found")
    except Exception as e:
        return Response(response=None, status=f"{INTERNAL} Directory not found")

@app.route("/api/set_default_examples", methods=["POST"])
def set_default_examples() -> Response:
    """Sets default files for each user."""
    try:
        # Should be called after auth is verified
        request_json = request.get_json()
        user_id = request_json.pop("user_id", DEFAULT_USER_ID)
        root_path = create_personal_folder(user_id)
        example_dir = os.path.dirname(os.path.dirname(app.config["UPLOAD_FOLDER"]))
        example_path = os.path.join(example_dir, "data/examples/")
        if os.path.exists(example_path):
            shutil.copytree(example_path, root_path, dirs_exist_ok=True)
            return jsonify(
                {"success": True, "message": "Default examples are set successfully"})
        else:
            return Response(response=None,
                            status=f"{UNFOUND} Directory not found at {example_dir}")
    except Exception as e:
        return Response(response=None,
                        status=f"{INTERNAL} Fail to Set Default Examples")


    message_pool,
)
from backend.schemas import DEFAULT_USER_ID
from backend.utils.utils import create_personal_folder
from backend.utils.user_conversation_storage import get_user_conversation_storage
from backend.utils.utils import (
    allowed_file,
    get_data_model_cls,
    get_user_and_chat_id_from_request,
    get_user_and_chat_id_from_request_json,
    is_sqlite_file,
    is_table_file,
    is_image_file,
    load_grounding_source,
)
from backend.schemas import INTERNAL, UNFOUND

TABLE_HUMAN_SIDE_FORMAT = "material-react-table"

def _path_tree_for_react_dnd_treeview(tree: list, id_to_path_dict: dict, path: str,
                                      parent: int,
                                      highlighted_files: list = []) -> list:
    """
    {
        "id": 1,
        "parent": 0,
        "droppable": true,
        "text": "Folder 1"
    },
    {
        "id": 2,
        "parent": 1,
        "text": "File 1-1",
        "data": {
            "fileType": "csv",
            "fileSize": "0.5MB"
        }
    },
    """
    for item in os.listdir(path):
        if item.startswith("."):
            continue
        item_path = os.path.join(path, item)
        droppable = os.path.isdir(item_path)
        idx = len(tree) + 1
        tree.append({
            "id": idx,
            "parent": parent,
            "droppable": droppable,
            "text": item,
            "highlight": True if item_path in highlighted_files else False})
        id_to_path_dict[idx] = item_path
        if os.path.isdir(item_path):
            _path_tree_for_react_dnd_treeview(tree, id_to_path_dict, item_path, idx)

    return []

def secure_filename(filename: str) -> str:
    keep_characters = ('.', '_')
    filename = "".join(
        c for c in filename if c.isalnum() or c in keep_characters).rstrip()
    return filename

@app.route("/api/upload", methods=["POST"])
def create_upload_file() -> dict | Response:
    """Uploads a new file."""
    try:
        if "file" not in request.files:
            return {"error": "No file part in the request"}
        file = request.files["file"]
        (user_id, chat_id) = get_user_and_chat_id_from_request(request)
        folder = create_personal_folder(user_id)

        # Check if the file is allowed
        if not allowed_file(str(file.filename)):


    },
    """
    for item in os.listdir(path):
        if item.startswith("."):
            continue
        item_path = os.path.join(path, item)
        droppable = os.path.isdir(item_path)
        idx = len(tree) + 1
        tree.append({
            "id": idx,
            "parent": parent,
            "droppable": droppable,
            "text": item,
            "highlight": True if item_path in highlighted_files else False})
        id_to_path_dict[idx] = item_path
        if os.path.isdir(item_path):
            _path_tree_for_react_dnd_treeview(tree, id_to_path_dict, item_path, idx)

    return []

def secure_filename(filename: str) -> str:
    keep_characters = ('.', '_')
    filename = "".join(
        c for c in filename if c.isalnum() or c in keep_characters).rstrip()
    return filename

@app.route("/api/upload", methods=["POST"])
def create_upload_file() -> dict | Response:
    """Uploads a new file."""
    try:
        if "file" not in request.files:
            return {"error": "No file part in the request"}
        file = request.files["file"]
        (user_id, chat_id) = get_user_and_chat_id_from_request(request)
        folder = create_personal_folder(user_id)

        # Check if the file is allowed
        if not allowed_file(str(file.filename)):
            return {"error": "File type not allowed"}

        # Save and read the file
        file.filename = secure_filename(str(file.filename))
        file_path = os.path.join(folder, file.filename)
        file.save(file_path)


        tree.append({
            "id": idx,
            "parent": parent,
            "droppable": droppable,
            "text": item,
            "highlight": True if item_path in highlighted_files else False})
        id_to_path_dict[idx] = item_path
        if os.path.isdir(item_path):
            _path_tree_for_react_dnd_treeview(tree, id_to_path_dict, item_path, idx)

    return []

def secure_filename(filename: str) -> str:
    keep_characters = ('.', '_')
    filename = "".join(
        c for c in filename if c.isalnum() or c in keep_characters).rstrip()
    return filename

@app.route("/api/upload", methods=["POST"])
def create_upload_file() -> dict | Response:
    """Uploads a new file."""
    try:
        if "file" not in request.files:
            return {"error": "No file part in the request"}
        file = request.files["file"]
        (user_id, chat_id) = get_user_and_chat_id_from_request(request)
        folder = create_personal_folder(user_id)

        # Check if the file is allowed
        if not allowed_file(str(file.filename)):
            return {"error": "File type not allowed"}

        # Save and read the file
        file.filename = secure_filename(str(file.filename))
        file_path = os.path.join(folder, file.filename)
        file.save(file_path)
        response = {"success": file.filename}

        logger.bind(user_id=user_id, chat_id=chat_id, api="/upload",
                    msg_head="Upload file success").debug(file_path)

        return jsonify(response)
    except Exception as e:
        logger.bind(user_id=user_id, chat_id=chat_id, api="/upload",
                    msg_head="Upload file error").error(str(e))

        return Response(response=None, status=f"{INTERNAL} Upload File Error: {str(e)}")

def _generate_human_side_data_from_file(filename: str, data_model: Any) -> Dict:
    if is_table_file(filename):
        # Determine the format of the human side(frontend) table
        human_side_data = data_model.get_human_side_data(mode="FULL")
        if TABLE_HUMAN_SIDE_FORMAT == "markdown":
            human_side_data = human_side_data.to_markdown(index=False)
            human_side_data_type = "plain"
        elif TABLE_HUMAN_SIDE_FORMAT == "material-react-table":
            columns = list(map(lambda item: {"accessorKey": item, "header": item},
                               human_side_data.columns.tolist()))
            data = human_side_data.fillna("").to_dict(orient="records")
            human_side_data = json.dumps({"columns": columns, "data": data})
            human_side_data_type = "table"
        data = {"success": filename, "content": human_side_data,
                "type": human_side_data_type}
    elif is_sqlite_file(filename):
        data = {"success": filename, "content": filename, "type": "table"}
    elif is_image_file(filename):


        if not allowed_file(str(file.filename)):
            return {"error": "File type not allowed"}

        # Save and read the file
        file.filename = secure_filename(str(file.filename))
        file_path = os.path.join(folder, file.filename)
        file.save(file_path)
        response = {"success": file.filename}

        logger.bind(user_id=user_id, chat_id=chat_id, api="/upload",
                    msg_head="Upload file success").debug(file_path)

        return jsonify(response)
    except Exception as e:
        logger.bind(user_id=user_id, chat_id=chat_id, api="/upload",
                    msg_head="Upload file error").error(str(e))

        return Response(response=None, status=f"{INTERNAL} Upload File Error: {str(e)}")

def _generate_human_side_data_from_file(filename: str, data_model: Any) -> Dict:
    if is_table_file(filename):
        # Determine the format of the human side(frontend) table
        human_side_data = data_model.get_human_side_data(mode="FULL")
        if TABLE_HUMAN_SIDE_FORMAT == "markdown":
            human_side_data = human_side_data.to_markdown(index=False)
            human_side_data_type = "plain"
        elif TABLE_HUMAN_SIDE_FORMAT == "material-react-table":
            columns = list(map(lambda item: {"accessorKey": item, "header": item},
                               human_side_data.columns.tolist()))
            data = human_side_data.fillna("").to_dict(orient="records")
            human_side_data = json.dumps({"columns": columns, "data": data})
            human_side_data_type = "table"
        data = {"success": filename, "content": human_side_data,
                "type": human_side_data_type}
    elif is_sqlite_file(filename):
        data = {"success": filename, "content": filename, "type": "table"}
    elif is_image_file(filename):
        # Determine the format of human side(frontend) image
        human_side_data = data_model.get_human_side_data()
        data = {"success": filename, "content": human_side_data, "type": "image"}
    else:
        return {"error": "Document file type not supported"}
    return data

def _get_file_path_from_node(folder: str, file_node: dict) -> Any:
    path_tree_list: list = []
    id_to_path_dict = {0: folder}
    _path_tree_for_react_dnd_treeview(path_tree_list, id_to_path_dict, folder, 0)
    path = id_to_path_dict[file_node["id"]]
    return path

@app.route("/api/file_system/apply", methods=["POST"])
def apply_to_conversation() -> Response:
    """Applies data to the conversation."""
    try:
        request_json = request.get_json()
        (user_id, chat_id) = get_user_and_chat_id_from_request_json(request_json)
        file_node = request_json["activated_file"]
        parent_message_id = request_json["parent_message_id"]
        folder = create_personal_folder(user_id)


            human_side_data_type = "plain"
        elif TABLE_HUMAN_SIDE_FORMAT == "material-react-table":
            columns = list(map(lambda item: {"accessorKey": item, "header": item},
                               human_side_data.columns.tolist()))
            data = human_side_data.fillna("").to_dict(orient="records")
            human_side_data = json.dumps({"columns": columns, "data": data})
            human_side_data_type = "table"
        data = {"success": filename, "content": human_side_data,
                "type": human_side_data_type}
    elif is_sqlite_file(filename):
        data = {"success": filename, "content": filename, "type": "table"}
    elif is_image_file(filename):
        # Determine the format of human side(frontend) image
        human_side_data = data_model.get_human_side_data()
        data = {"success": filename, "content": human_side_data, "type": "image"}
    else:
        return {"error": "Document file type not supported"}
    return data

def _get_file_path_from_node(folder: str, file_node: dict) -> Any:
    path_tree_list: list = []
    id_to_path_dict = {0: folder}
    _path_tree_for_react_dnd_treeview(path_tree_list, id_to_path_dict, folder, 0)
    path = id_to_path_dict[file_node["id"]]
    return path

@app.route("/api/file_system/apply", methods=["POST"])
def apply_to_conversation() -> Response:
    """Applies data to the conversation."""
    try:
        request_json = request.get_json()
        (user_id, chat_id) = get_user_and_chat_id_from_request_json(request_json)
        file_node = request_json["activated_file"]
        parent_message_id = request_json["parent_message_id"]
        folder = create_personal_folder(user_id)

        # Modify the selected grounding sources
        grounding_source_dict = grounding_source_pool.get_pool_info_with_id(user_id,
                                                                            chat_id,
                                                                            default_value={})
        file_path = _get_file_path_from_node(folder, file_node)
        filename = file_node["text"]
        filename_no_ext = os.path.splitext(filename)[0]
        if file_path not in grounding_source_dict:


    elif is_sqlite_file(filename):
        data = {"success": filename, "content": filename, "type": "table"}
    elif is_image_file(filename):
        # Determine the format of human side(frontend) image
        human_side_data = data_model.get_human_side_data()
        data = {"success": filename, "content": human_side_data, "type": "image"}
    else:
        return {"error": "Document file type not supported"}
    return data

def _get_file_path_from_node(folder: str, file_node: dict) -> Any:
    path_tree_list: list = []
    id_to_path_dict = {0: folder}
    _path_tree_for_react_dnd_treeview(path_tree_list, id_to_path_dict, folder, 0)
    path = id_to_path_dict[file_node["id"]]
    return path

@app.route("/api/file_system/apply", methods=["POST"])
def apply_to_conversation() -> Response:
    """Applies data to the conversation."""
    try:
        request_json = request.get_json()
        (user_id, chat_id) = get_user_and_chat_id_from_request_json(request_json)
        file_node = request_json["activated_file"]
        parent_message_id = request_json["parent_message_id"]
        folder = create_personal_folder(user_id)

        # Modify the selected grounding sources
        grounding_source_dict = grounding_source_pool.get_pool_info_with_id(user_id,
                                                                            chat_id,
                                                                            default_value={})
        file_path = _get_file_path_from_node(folder, file_node)
        filename = file_node["text"]
        filename_no_ext = os.path.splitext(filename)[0]
        if file_path not in grounding_source_dict:
            data = load_grounding_source(file_path)
            data_model = get_data_model_cls(filename).from_raw_data(
                raw_data=data,
                raw_data_name=filename_no_ext,
                raw_data_path=file_path,
            )
            grounding_source_dict[file_path] = data_model
            # Add uploaded file in chat memory
            message_list = message_pool.get_pool_info_with_id(user_id, chat_id,
                                                              default_value=list())
            llm_side_data = data_model.get_llm_side_data()
            human_message_content = "[User uploaded a file {}]\n{}".format(filename,
                                                                           llm_side_data)
            human_message_id = message_id_register.add_variable(human_message_content)
            message_list.append(
                {
                    "message_id": human_message_id,
                    "parent_message_id": parent_message_id,
                    "message_type": "human_message",
                    "message_content": human_message_content,
                }
            )
            data = _generate_human_side_data_from_file(filename, data_model)
            message_pool.set_pool_info_with_id(user_id, chat_id, message_list)
            grounding_source_pool.set_pool_info_with_id(user_id, chat_id,
                                                        grounding_source_dict)
            # Dump to database
            db = get_user_conversation_storage()
            db_message = {
                "conversation_id": chat_id,
                "user_id": user_id,
                "message_id": human_message_id,
                "parent_message_id": parent_message_id,
                "version_id": 0,
                "role": "user",
                "data_for_human": {
                    "intermediate_steps": [],
                    "final_answer": [
                        {
                            "type": data["type"],
                            "text": data["content"],
                            "final": True,
                        }
                    ],
                },
                "data_for_llm": message_list[-1]["message_content"],
                "raw_data": None,
            }
            db.message.insert_one(db_message)
            response = {
                "success": True,
                "message_id": human_message_id,
                "parent_message_id": parent_message_id,
                "message": "Successfully apply {} to conversation {}".format(filename,
                                                                             chat_id),
                "content": {
                    "intermediate_steps": [],
                    "final_answer": [
                        {
                            "type": data["type"],
                            "text": data["content"],
                            "final": True,
                        }
                    ],
                },
            }

            logger.bind(user_id=user_id, chat_id=chat_id, api="/apply",
                        msg_head="Apply file success").debug(file_path)
            del db_message["data_for_human"]

            return jsonify(response)
        else:
            logger.bind(user_id=user_id, chat_id=chat_id, api="/apply",
                        msg_head="Apply file failed").debug(file_path)

            return jsonify({"success": False,
                            "message": "You have already import {} to the conversation".format(
                                filename)})
    except Exception as e:
        logger.bind(user_id=user_id, chat_id=chat_id, api="/apply",
                    msg_head="Apply file failed").error(file_path)
        import traceback
        traceback.print_exc()

        return Response(response=None,
                        status=f"{INTERNAL} Fail to apply file to chat: {str(e)}")

@app.route("/api/file_system/move", methods=["POST"])
def move_files() -> Response:
    """Moves file from source path from target source."""
    request_json = request.get_json()
    (user_id, chat_id) = get_user_and_chat_id_from_request_json(request_json)
    root_path = create_personal_folder(user_id)
    nodes = request_json["nodes"]
    try:
        if os.path.exists(root_path) and os.path.isdir(root_path):
            current_path_tree_list: list = []
            id_to_path_dict = {0: root_path}
            _path_tree_for_react_dnd_treeview(current_path_tree_list, id_to_path_dict,
                                              root_path, 0)
        for node in nodes:
            old_path = id_to_path_dict[node["id"]]
            new_path = id_to_path_dict[node["parent"]]
            shutil.move(old_path, new_path)


            return jsonify(response)
        else:
            logger.bind(user_id=user_id, chat_id=chat_id, api="/apply",
                        msg_head="Apply file failed").debug(file_path)

            return jsonify({"success": False,
                            "message": "You have already import {} to the conversation".format(
                                filename)})
    except Exception as e:
        logger.bind(user_id=user_id, chat_id=chat_id, api="/apply",
                    msg_head="Apply file failed").error(file_path)
        import traceback
        traceback.print_exc()

        return Response(response=None,
                        status=f"{INTERNAL} Fail to apply file to chat: {str(e)}")

@app.route("/api/file_system/move", methods=["POST"])
def move_files() -> Response:
    """Moves file from source path from target source."""
    request_json = request.get_json()
    (user_id, chat_id) = get_user_and_chat_id_from_request_json(request_json)
    root_path = create_personal_folder(user_id)
    nodes = request_json["nodes"]
    try:
        if os.path.exists(root_path) and os.path.isdir(root_path):
            current_path_tree_list: list = []
            id_to_path_dict = {0: root_path}
            _path_tree_for_react_dnd_treeview(current_path_tree_list, id_to_path_dict,
                                              root_path, 0)
        for node in nodes:
            old_path = id_to_path_dict[node["id"]]
            new_path = id_to_path_dict[node["parent"]]
            shutil.move(old_path, new_path)

            logger.bind(user_id=user_id, chat_id=chat_id, api="/move",
                        msg_head="Move file success").debug(
                f"from {old_path} to {new_path}"
            )

            return jsonify({"success": True, "message": "File moved successfully"})
    except Exception as e:
        logger.bind(user_id=user_id, chat_id=chat_id, api="/move",
                    msg_head="Move file failed").error(str(e))

        return jsonify({"success": False, "message": str(e)})
    return Response(response=None, status=f"{INTERNAL} Fail to move file")

@app.route("/api/file_system/delete", methods=["POST"])
def delete_files() -> Response:
    """Deletes a file from the filesystem."""
    request_json = request.get_json()
    (user_id, chat_id) = get_user_and_chat_id_from_request_json(request_json)
    root_path = create_personal_folder(user_id)
    node = request_json["node"]
    try:
        if os.path.exists(root_path) and os.path.isdir(root_path):
            current_path_tree_list: list = []
            id_to_path_dict = {0: root_path}
            _path_tree_for_react_dnd_treeview(current_path_tree_list, id_to_path_dict,
                                              root_path, 0)
            path = id_to_path_dict[node["id"]]
            if os.path.isdir(path):
                shutil.rmtree(path)
            else:
                os.remove(path)


        for node in nodes:
            old_path = id_to_path_dict[node["id"]]
            new_path = id_to_path_dict[node["parent"]]
            shutil.move(old_path, new_path)

            logger.bind(user_id=user_id, chat_id=chat_id, api="/move",
                        msg_head="Move file success").debug(
                f"from {old_path} to {new_path}"
            )

            return jsonify({"success": True, "message": "File moved successfully"})
    except Exception as e:
        logger.bind(user_id=user_id, chat_id=chat_id, api="/move",
                    msg_head="Move file failed").error(str(e))

        return jsonify({"success": False, "message": str(e)})
    return Response(response=None, status=f"{INTERNAL} Fail to move file")

@app.route("/api/file_system/delete", methods=["POST"])
def delete_files() -> Response:
    """Deletes a file from the filesystem."""
    request_json = request.get_json()
    (user_id, chat_id) = get_user_and_chat_id_from_request_json(request_json)
    root_path = create_personal_folder(user_id)
    node = request_json["node"]
    try:
        if os.path.exists(root_path) and os.path.isdir(root_path):
            current_path_tree_list: list = []
            id_to_path_dict = {0: root_path}
            _path_tree_for_react_dnd_treeview(current_path_tree_list, id_to_path_dict,
                                              root_path, 0)
            path = id_to_path_dict[node["id"]]
            if os.path.isdir(path):
                shutil.rmtree(path)
            else:
                os.remove(path)

        logger.bind(user_id=user_id, chat_id=chat_id, api="/delete",
                    msg_head="Delete file success").debug(path)

        return jsonify({"success": True, "message": "File is deleted successfully"})
    except Exception as e:
        logger.bind(user_id=user_id, chat_id=chat_id, api="/delete",
                    msg_head="Delete file failed").error(str(e))

        return Response(response=None,
                        status=f"{INTERNAL} Delete file failed: {str(e)}")

@app.route("/api/file_system/download", methods=["POST"])
def download_files() -> Response:
    """Downloads a file to local."""
    request_json = request.get_json()
    user_id = request_json.pop("user_id", DEFAULT_USER_ID)
    root_path = create_personal_folder(user_id)
    node = request_json["node"]

    try:
        if os.path.exists(root_path) and os.path.isdir(root_path):
            current_path_tree_list: list = []
            id_to_path_dict = {0: root_path}
            _path_tree_for_react_dnd_treeview(current_path_tree_list, id_to_path_dict,
                                              root_path, 0)
            path = id_to_path_dict[node["id"]]

            if os.path.exists(path):
                logger.bind(user_id=user_id, api="/download",


                                              root_path, 0)
            path = id_to_path_dict[node["id"]]
            if os.path.isdir(path):
                shutil.rmtree(path)
            else:
                os.remove(path)

        logger.bind(user_id=user_id, chat_id=chat_id, api="/delete",
                    msg_head="Delete file success").debug(path)

        return jsonify({"success": True, "message": "File is deleted successfully"})
    except Exception as e:
        logger.bind(user_id=user_id, chat_id=chat_id, api="/delete",
                    msg_head="Delete file failed").error(str(e))

        return Response(response=None,
                        status=f"{INTERNAL} Delete file failed: {str(e)}")

@app.route("/api/file_system/download", methods=["POST"])
def download_files() -> Response:
    """Downloads a file to local."""
    request_json = request.get_json()
    user_id = request_json.pop("user_id", DEFAULT_USER_ID)
    root_path = create_personal_folder(user_id)
    node = request_json["node"]

    try:
        if os.path.exists(root_path) and os.path.isdir(root_path):
            current_path_tree_list: list = []
            id_to_path_dict = {0: root_path}
            _path_tree_for_react_dnd_treeview(current_path_tree_list, id_to_path_dict,
                                              root_path, 0)
            path = id_to_path_dict[node["id"]]

            if os.path.exists(path):
                logger.bind(user_id=user_id, api="/download",
                            msg_head="download file success").debug(path)
                return send_file(path, as_attachment=True)

        logger.bind(user_id=user_id, api="/download",
                    msg_head="download file failed").debug(path)
        return Response(response=None,
                        status=f"{INTERNAL} Download file failed: file not correctlt sent")

    except Exception as e:
        print(str(e))
        import traceback
        traceback.print_exc()

        logger.bind(user_id=user_id, api="/download",
                    msg_head="download file failed").error(str(e))

        return Response(response=None,
                        status=f"{INTERNAL} Download file failed: {str(e)}")

def _generate_directory_name(name: str, x:int=0) -> Any:
    dir_name = (name + ("_" + str(x) if x != 0 else "")).strip()
    if not os.path.exists(dir_name):
        return dir_name
    else:
        return _generate_directory_name(name, x + 1)

@app.route("/api/file_system/create_folder", methods=["POST"])
def create_folders() -> Response:
    """Creates a folder in the filesystem."""
    request_json = request.get_json()
    (user_id, chat_id) = get_user_and_chat_id_from_request_json(request_json)
    root_path = create_personal_folder(user_id)
    if os.path.exists(root_path) and os.path.isdir(root_path):
        try:
            new_path = _generate_directory_name(os.path.join(root_path, "Folder"))
            os.makedirs(new_path, exist_ok=False)