Upload 2846 files

.gitattributes

@@ -33,3 +33,17 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+examples/antibiotic_pred/EC_antibiotic.xlsx filter=lfs diff=lfs merge=lfs -text
+examples/antibiotic_pred/ec_train filter=lfs diff=lfs merge=lfs -text
+examples/antibiotic_pred/ec_train.json filter=lfs diff=lfs merge=lfs -text
+examples/hotpotqa/test[[:space:]]structopt[[:space:]]toolcall.ipynb filter=lfs diff=lfs merge=lfs -text
+examples/pertqa/.ipynb_checkpoints/test[[:space:]]structual[[:space:]]ourloop-withsearch-checkpoint.ipynb filter=lfs diff=lfs merge=lfs -text
+examples/pertqa/k562_processed_grn.csv filter=lfs diff=lfs merge=lfs -text
+examples/pertqa/pert_folder/EGRET_K562.csv filter=lfs diff=lfs merge=lfs -text
+examples/pertqa/reploge_train.json filter=lfs diff=lfs merge=lfs -text
+examples/pertqa/replogle_update_train.csv filter=lfs diff=lfs merge=lfs -text
+examples/pertqa/replogle_update_train.json filter=lfs diff=lfs merge=lfs -text
+examples/pertqa/test[[:space:]]structual[[:space:]]ourloop-withsearch.ipynb filter=lfs diff=lfs merge=lfs -text
+examples/pubmedqa/pubmedqa_train.json filter=lfs diff=lfs merge=lfs -text
+examples/workflow/invest/300750/20250815/graphs/300750_candlestick_chart.png filter=lfs diff=lfs merge=lfs -text
+examples/workflow/invest/300750/20250815/graphs/300750_technical_charts.png filter=lfs diff=lfs merge=lfs -text

evoagentx/.ipynb_checkpoints/test aflow-checkpoint.ipynb

@@ -0,0 +1,427 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "e2d3caf8",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "import matplotlib.pyplot as plt\n",
+    "import pickle\n",
+    "import glob\n",
+    "import pandas as pd\n",
+    "import glob\n",
+    "from tqdm import tqdm\n",
+    "import base64\n",
+    "import requests\n",
+    "# OpenAI API Key\n",
+    "api_key = \"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\"\n",
+    "# Function to encode the image\n",
+    "import os\n",
+    "os.environ[\"OPENAI_API_KEY\"] = api_key\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "f870b639",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/gpfs/radev/home/tl688/.conda/envs/evoagentx/lib/python3.11/site-packages/PyPDF2/__init__.py:21: DeprecationWarning: PyPDF2 is deprecated. Please move to the pypdf library instead.\n",
+      "  warnings.warn(\n"
+     ]
+    }
+   ],
+   "source": [
+    "import os\n",
+    "from dotenv import load_dotenv\n",
+    "from evoagentx.optimizers import AFlowOptimizer\n",
+    "from evoagentx.models import LiteLLMConfig, LiteLLM, OpenAILLMConfig, OpenAILLM\n",
+    "from evoagentx.benchmark import AFlowHumanEval\n",
+    "\n",
+    "# Load environment variables\n",
+    "load_dotenv()\n",
+    "OPENAI_API_KEY = os.getenv(\"OPENAI_API_KEY\")\n",
+    "# ANTHROPIC_API_KEY = os.getenv(\"ANTHROPIC_API_KEY\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "1f3dd892",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# # Configure the optimizer LLM (Claude 3.5 Sonnet)\n",
+    "# claude_config = LiteLLMConfig(\n",
+    "#     model=\"anthropic/claude-3-5-sonnet-20240620\", \n",
+    "#     anthropic_key=ANTHROPIC_API_KEY\n",
+    "# )\n",
+    "# optimizer_llm = LiteLLM(config=claude_config)\n",
+    "\n",
+    "# Configure the executor LLM (GPT-4o-mini)\n",
+    "openai_config = OpenAILLMConfig(\n",
+    "    model=\"gpt-4o-mini\", \n",
+    "    openai_key=OPENAI_API_KEY\n",
+    ")\n",
+    "\n",
+    "claude_config = LiteLLMConfig(\n",
+    "    model=\"gpt-4o-mini\", \n",
+    "    openai_key=OPENAI_API_KEY\n",
+    ")\n",
+    "executor_llm = OpenAILLM(config=openai_config)\n",
+    "optimizer_llm = LiteLLM(config=claude_config)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "a87feb08",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "EXPERIMENTAL_CONFIG = {\n",
+    "    \"humaneval\": {\n",
+    "        \"question_type\": \"code\", \n",
+    "        \"operators\": [\"Custom\", \"CustomCodeGenerate\", \"Test\", \"ScEnsemble\"] \n",
+    "    }, \n",
+    "    \"mbpp\": {\n",
+    "        \"question_type\": \"code\", \n",
+    "        \"operators\": [\"Custom\", \"CustomCodeGenerate\", \"Test\", \"ScEnsemble\"] \n",
+    "    },\n",
+    "    \"hotpotqa\": {\n",
+    "        \"question_type\": \"qa\", \n",
+    "        \"operators\": [\"Custom\", \"AnswerGenerate\", \"QAScEnsemble\"]\n",
+    "    },\n",
+    "    \"gsm8k\": {\n",
+    "        \"question_type\": \"math\", \n",
+    "        \"operators\": [\"Custom\", \"ScEnsemble\", \"Programmer\"]\n",
+    "    },\n",
+    "    \"math\": {\n",
+    "        \"question_type\": \"math\", \n",
+    "        \"operators\": [\"Custom\", \"ScEnsemble\", \"Programmer\"]\n",
+    "    }\n",
+    "}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "b6054068",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import evoagentx.workflow.operators as operator\n",
+    "import examples.aflow.code_generation.prompt as prompt_custom # noqa: F401\n",
+    "from evoagentx.models.model_configs import LLMConfig\n",
+    "from evoagentx.benchmark.benchmark import Benchmark\n",
+    "from evoagentx.models.model_utils import create_llm_instance\n",
+    "\n",
+    "class Workflow:\n",
+    "    \n",
+    "    def __init__(\n",
+    "        self,\n",
+    "        name: str,\n",
+    "        llm_config: LLMConfig,\n",
+    "        benchmark: Benchmark\n",
+    "    ):\n",
+    "        self.name = name\n",
+    "        self.llm = create_llm_instance(llm_config)\n",
+    "        self.benchmark = benchmark \n",
+    "        self.custom = operator.Custom(self.llm)\n",
+    "        self.custom_code_generate = operator.CustomCodeGenerate(self.llm)\n",
+    "\n",
+    "    async def __call__(self, problem: str, entry_point: str):\n",
+    "        \"\"\"\n",
+    "        Implementation of the workflow\n",
+    "        Custom operator to generate anything you want.\n",
+    "        But when you want to get standard code, you should use custom_code_generate operator.\n",
+    "        \"\"\"\n",
+    "        # await self.custom(input=, instruction=\"\")\n",
+    "        solution = await self.custom_code_generate(problem=problem, entry_point=entry_point, instruction=prompt_custom.GENERATE_PYTHON_CODE_PROMPT) # But When you want to get standard code ,you should use customcodegenerator.\n",
+    "        return solution['response']"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "27e574ad",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\u001b[32m2025-10-12 15:04:04.523\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.benchmark.humaneval\u001b[0m:\u001b[36m_load_data\u001b[0m:\u001b[36m182\u001b[0m - \u001b[1mLoading train data from None\u001b[0m\n",
+      "\u001b[32m2025-10-12 15:04:04.524\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.benchmark.humaneval\u001b[0m:\u001b[36m_load_data\u001b[0m:\u001b[36m185\u001b[0m - \u001b[1mLoading dev data from humaneval_validate.jsonl\u001b[0m\n",
+      "\u001b[32m2025-10-12 15:04:04.525\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.benchmark.humaneval\u001b[0m:\u001b[36m_load_data\u001b[0m:\u001b[36m188\u001b[0m - \u001b[1mLoading test data from humaneval_test.jsonl\u001b[0m\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Initialize the benchmark\n",
+    "humaneval = AFlowHumanEval()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "2f8da181",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "optimizer = AFlowOptimizer(\n",
+    "    graph_path=\"../examples/aflow/code_generation\",  # Path to the initial workflow graph\n",
+    "    optimized_path=\"../examples/aflow/humaneval/optimized\",  # Path to save optimized workflows\n",
+    "    optimizer_llm=optimizer_llm,  # LLM for optimization\n",
+    "    executor_llm=executor_llm,    # LLM for execution\n",
+    "    validation_rounds=3,          # Number of times to run validation on the development set during optimization\n",
+    "    eval_rounds=3,               # Number of times to run evaluation on the test set during testing\n",
+    "    max_rounds=20,               # Maximum optimization rounds\n",
+    "    **EXPERIMENTAL_CONFIG[\"humaneval\"]  # Task-specific configuration, used to specify the task type and available operators\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "74937699",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import nest_asyncio\n",
+    "nest_asyncio.apply()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "98ac4a63",
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\u001b[32m2025-10-12 15:04:50.304\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.utils.aflow_utils.graph_utils\u001b[0m:\u001b[36mload_graph\u001b[0m:\u001b[36m51\u001b[0m - \u001b[1mError loading graph for round 0: No module named '.'\u001b[0m\n",
+      "\u001b[32m2025-10-12 15:04:50.305\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.aflow_optimizer\u001b[0m:\u001b[36m_execute_with_retry\u001b[0m:\u001b[36m147\u001b[0m - \u001b[1mError occurred: No module named '.'. Retrying... (Attempt 1/3)\u001b[0m\n",
+      "\u001b[32m2025-10-12 15:04:55.310\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.utils.aflow_utils.graph_utils\u001b[0m:\u001b[36mload_graph\u001b[0m:\u001b[36m51\u001b[0m - \u001b[1mError loading graph for round 0: No module named '.'\u001b[0m\n",
+      "\u001b[32m2025-10-12 15:04:55.311\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.aflow_optimizer\u001b[0m:\u001b[36m_execute_with_retry\u001b[0m:\u001b[36m147\u001b[0m - \u001b[1mError occurred: No module named '.'. Retrying... (Attempt 2/3)\u001b[0m\n",
+      "\u001b[32m2025-10-12 15:05:05.322\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.utils.aflow_utils.graph_utils\u001b[0m:\u001b[36mload_graph\u001b[0m:\u001b[36m51\u001b[0m - \u001b[1mError loading graph for round 0: No module named '.'\u001b[0m\n",
+      "\u001b[32m2025-10-12 15:05:05.322\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.aflow_optimizer\u001b[0m:\u001b[36m_execute_with_retry\u001b[0m:\u001b[36m147\u001b[0m - \u001b[1mError occurred: No module named '.'. Retrying... (Attempt 3/3)\u001b[0m\n",
+      "\u001b[32m2025-10-12 15:05:05.322\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.aflow_optimizer\u001b[0m:\u001b[36m_execute_with_retry\u001b[0m:\u001b[36m149\u001b[0m - \u001b[1mMax retries reached.\u001b[0m\n",
+      "\u001b[32m2025-10-12 15:05:05.323\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.aflow_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m112\u001b[0m - \u001b[1mScore for round 1: None\u001b[0m\n",
+      "\u001b[32m2025-10-12 15:05:05.326\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.aflow_optimizer\u001b[0m:\u001b[36m_execute_with_retry\u001b[0m:\u001b[36m147\u001b[0m - \u001b[1mError occurred: 'round'. Retrying... (Attempt 1/3)\u001b[0m\n",
+      "\u001b[32m2025-10-12 15:05:10.332\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.aflow_optimizer\u001b[0m:\u001b[36m_execute_with_retry\u001b[0m:\u001b[36m147\u001b[0m - \u001b[1mError occurred: 'round'. Retrying... (Attempt 2/3)\u001b[0m\n",
+      "\u001b[32m2025-10-12 15:05:20.344\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.aflow_optimizer\u001b[0m:\u001b[36m_execute_with_retry\u001b[0m:\u001b[36m147\u001b[0m - \u001b[1mError occurred: 'round'. Retrying... (Attempt 3/3)\u001b[0m\n",
+      "\u001b[32m2025-10-12 15:05:20.344\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.aflow_optimizer\u001b[0m:\u001b[36m_execute_with_retry\u001b[0m:\u001b[36m149\u001b[0m - \u001b[1mMax retries reached.\u001b[0m\n",
+      "\u001b[32m2025-10-12 15:05:20.345\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.aflow_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m112\u001b[0m - \u001b[1mScore for round 2: None\u001b[0m\n",
+      "\u001b[32m2025-10-12 15:05:20.347\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mevoagentx.optimizers.aflow_optimizer\u001b[0m:\u001b[36m_execute_with_retry\u001b[0m:\u001b[36m147\u001b[0m - \u001b[1mError occurred: 'round'. Retrying... (Attempt 1/3)\u001b[0m\n"
+     ]
+    }
+   ],
+   "source": [
+    "# Optimize the workflow\n",
+    "optimizer.optimize(humaneval)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1010d583",
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [],
+   "source": [
+    "optimizer.test(humaneval)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "becb5a82",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "id": "5c076d29",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df = pd.read_json(\"/home/tl688/pitl688/selfevolve/AFlow/data/datasets/scicode_dev.jsonl\", lines=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "id": "481602a9",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'def get_alpha(recvec, alpha_scaling=5):\\n    \"\"\"\\n    Calculate the alpha value for the Ewald summation, scaled by a specified factor.\\n    Parameters:\\n        recvec (np.ndarray): A 3x3 array representing the reciprocal lattice vectors.\\n        alpha_scaling (float): A scaling factor applied to the alpha value. Default is 5.\\n    Returns:\\n        float: The calculated alpha value.\\n    \"\"\"\\n    alpha = alpha_scaling * np.max(np.linalg.norm(recvec, axis=1))\\n    return alpha'"
+      ]
+     },
+     "execution_count": 23,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df['ground_truth_code'].values[0]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "id": "ffb0be7e",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "\"def get_alpha(recvec, alpha_scaling=5):\\n    '''Calculate the alpha value for the Ewald summation, scaled by a specified factor.\\n    Parameters:\\n        recvec (np.ndarray): A 3x3 array representing the reciprocal lattice vectors.\\n        alpha_scaling (float): A scaling factor applied to the alpha value. Default is 5.\\n    Returns:\\n        float: The calculated alpha value.\\n    '''\""
+      ]
+     },
+     "execution_count": 21,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df['function_header'].values[0]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 24,
+   "id": "69acf613",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'import numpy as np\\nfrom scipy.special import erfc'"
+      ]
+     },
+     "execution_count": 24,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df['required_dependencies'].values[0]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 25,
+   "id": "b5696e0e",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Index(['step_number', 'step_description_prompt', 'step_background',\n",
+       "       'ground_truth_code', 'function_header', 'test_cases', 'return_line',\n",
+       "       'required_dependencies'],\n",
+       "      dtype='object')"
+      ]
+     },
+     "execution_count": 25,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df.columns"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 27,
+   "id": "0a3085a9",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "\"def get_alpha(recvec, alpha_scaling=5):\\n    '''Calculate the alpha value for the Ewald summation, scaled by a specified factor.\\n    Parameters:\\n        recvec (np.ndarray): A 3x3 array representing the reciprocal lattice vectors.\\n        alpha_scaling (float): A scaling factor applied to the alpha value. Default is 5.\\n    Returns:\\n        float: The calculated alpha value.\\n    '''\""
+      ]
+     },
+     "execution_count": 27,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df['function_header'].values[0]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 28,
+   "id": "e6a76c86",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "\"ref1 = -1.74756\\nEX1 = {\\n    'latvec': np.array([\\n        [0.0, 1.0, 1.0],\\n        [1.0, 0.0, 1.0],\\n        [1.0, 1.0, 0.0]\\n        ]),\\n    'atom_charges': np.array([1]),\\n    'atom_coords': np.array([\\n        [0.0, 0.0, 0.0]\\n        ]),\\n    'configs': np.array([\\n        [1.0, 1.0, 1.0]\\n    ]),\\n}\\nassert np.allclose(get_alpha(np.linalg.inv(EX1['latvec']).T), target)\""
+      ]
+     },
+     "execution_count": 28,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df['test_cases'].values[0]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "99775141",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.13"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

evoagentx/__init__.py

@@ -0,0 +1,2 @@
+
+__version__ = '0.1.0'

evoagentx/actions/__init__.py

@@ -0,0 +1,5 @@
+from .action import Action, ActionInput, ActionOutput
+from .code_verification import CodeVerification
+from .code_extraction import CodeExtraction
+
+__all__ = ["Action", "ActionInput", "ActionOutput", "CodeVerification", "CodeExtraction"]

evoagentx/actions/action.py

@@ -0,0 +1,256 @@
+import json
+from pydantic import model_validator 
+from pydantic_core import PydanticUndefined
+from typing import Optional, Type, Tuple, Union, List, Any
+
+from ..core.module import BaseModule
+from ..core.module_utils import get_type_name
+from ..core.registry import MODULE_REGISTRY
+# from ..core.base_config import Parameter
+from ..core.parser import Parser
+from ..core.message import Message
+from ..models.base_model import BaseLLM, LLMOutputParser
+from ..tools.tool import Toolkit 
+from ..prompts.context_extraction import CONTEXT_EXTRACTION
+from ..prompts.template import PromptTemplate  
+
+
+class ActionInput(LLMOutputParser):
+    """Input specification and parsing for actions.
+    
+    This class defines the input requirements for actions and provides methods
+    to generate structured input specifications. It inherits from LLMOutputParser 
+    to allow parsing of LLM outputs into structured inputs for actions.
+    
+    Notes:
+        Parameters in ActionInput should be defined in Pydantic Field format.
+        For optional variables, use format: 
+        var: Optional[int] = Field(default=None, description="xxx")
+        Remember to add `default=None` for optional parameters.
+    """
+
+    @classmethod
+    def get_input_specification(cls, ignore_fields: List[str] = []) -> str:
+        """Generate a JSON specification of the input requirements.
+        
+        Examines the class fields and produces a structured specification of
+        the input parameters, including their types, descriptions, and whether
+        they are required.
+        
+        Args:
+            ignore_fields (List[str]): List of field names to exclude from the specification.
+            
+        Returns:
+            A JSON string containing the input specification, or an empty string
+            if no fields are defined or all are ignored.
+        """
+        fields_info = {}
+        attrs = cls.get_attrs()
+        for field_name, field_info in cls.model_fields.items():
+            if field_name in ignore_fields:
+                continue
+            if field_name not in attrs:
+                continue
+            field_type = get_type_name(field_info.annotation)
+            field_desc = field_info.description if field_info.description is not None else None
+            # field_required = field_info.is_required()
+            field_default = str(field_info.default) if field_info.default is not PydanticUndefined else None
+            field_required = True if field_default is None else False
+            description = field_type + ", "
+            if field_desc is not None:
+                description += (field_desc.strip() + ", ") 
+            description += ("required" if field_required else "optional")
+            if field_default is not None:
+                description += (", Default value: " + field_default)
+            fields_info[field_name] = description
+        
+        if len(fields_info) == 0:
+            return "" 
+        fields_info_str = json.dumps(fields_info, indent=4)
+        return fields_info_str
+        
+    @classmethod
+    def get_required_input_names(cls) -> List[str]:
+        """Get a list of all required input parameter names.
+        
+        Returns:
+            List[str]: Names of all parameters that are required (don't have default values).
+        """
+        required_fields = []
+        attrs = cls.get_attrs()
+        for field_name, field_info in cls.model_fields.items():
+            if field_name not in attrs:
+                continue
+            field_default = field_info.default
+            # A field is required if it doesn't have a default value
+            if field_default is PydanticUndefined:
+                required_fields.append(field_name)
+        return required_fields
+
+
+class ActionOutput(LLMOutputParser):
+    """Output representation for actions.
+    
+    This class handles the structured output of actions, providing methods
+    to convert the output to structured data. It inherits from LLMOutputParser
+    to support parsing of LLM outputs into structured action results.
+    """
+    
+    def to_str(self) -> str:
+        """Convert the output to a formatted JSON string.
+        
+        Returns:
+            A pretty-printed JSON string representation of the structured data.
+        """
+        return json.dumps(self.get_structured_data(), indent=4)
+    
+
+class Action(BaseModule):
+    """Base class for all actions in the EvoAgentX framework.
+    
+    Actions represent discrete operations that can be performed by agents.
+    They define inputs, outputs, and execution behavior, and can optionally
+    use tools to accomplish their tasks.
+    
+    Attributes:
+        name (str): Unique identifier for the action.
+        description (str): Human-readable description of what the action does.
+        prompt (Optional[str]): Optional prompt template for this action.
+        tools (Optional[List[Toolkit]]): Optional list of tools that can be used by this action.
+        inputs_format (Optional[Type[ActionInput]]): Optional class defining the expected input structure.
+        outputs_format (Optional[Type[Parser]]): Optional class defining the expected output structure.
+    """
+
+    name: str
+    description: str
+    prompt: Optional[str] = None
+    prompt_template: Optional[PromptTemplate] = None 
+    tools: Optional[List[Toolkit]] = None # specify the possible tool for the action
+    inputs_format: Optional[Type[ActionInput]] = None # specify the input format of the action
+    outputs_format: Optional[Type[Parser]] = None  # specify the possible structured output format
+
+    def init_module(self):
+        """Initialize the action module.
+        
+        This method is called after the action is instantiated.
+        Subclasses can override this to perform custom initialization.
+        """
+        pass 
+
+    def to_dict(self, exclude_none: bool = True, ignore: List[str] = [], **kwargs) -> dict:
+        """
+        Convert the action to a dictionary for saving.  
+        """
+        data = super().to_dict(exclude_none=exclude_none, ignore=ignore, **kwargs)
+        if self.inputs_format:
+            data["inputs_format"] = self.inputs_format.__name__ 
+        if self.outputs_format:
+            data["outputs_format"] = self.outputs_format.__name__ 
+        # TODO: customize serialization for the tools 
+        return data 
+    
+    @model_validator(mode="before")
+    @classmethod
+    def validate_data(cls, data: Any) -> Any:
+        if "inputs_format" in data and data["inputs_format"] and isinstance(data["inputs_format"], str):
+            # only used when loading from a file
+            data["inputs_format"] = MODULE_REGISTRY.get_module(data["inputs_format"])
+        if "outputs_format" in data and data["outputs_format"] and isinstance(data["outputs_format"], str):
+            # only used when loading from a file
+            data["outputs_format"] = MODULE_REGISTRY.get_module(data["outputs_format"])
+        # TODO: customize loading for the tools
+        return data 
+    
+    def execute(self, llm: Optional[BaseLLM] = None, inputs: Optional[dict] = None, sys_msg: Optional[str]=None, return_prompt: bool = False, **kwargs) -> Optional[Union[Parser, Tuple[Parser, str]]]:
+        """Execute the action to produce a result.
+        
+        This is the main entry point for executing an action. Subclasses must
+        implement this method to define the action's behavior.
+
+        Args:
+            llm (Optional[BaseLLM]): The LLM used to execute the action.
+            inputs (Optional[dict]): Input data for the action execution. The input data should be a dictionary that matches the input format of the provided prompt. 
+                For example, if the prompt contains a variable `{input_var}`, the `inputs` dictionary should have a key `input_var`, otherwise the variable will be set to empty string. 
+            sys_msg (Optional[str]): Optional system message for the LLM.
+            return_prompt (bool): Whether to return the complete prompt passed to the LLM.
+            **kwargs (Any): Additional keyword arguments for the execution.
+        
+        Returns:
+            If `return_prompt` is False, the method returns a Parser object containing the structured result of the action.
+            If `return_prompt` is True, the method returns a tuple containing the Parser object and the complete prompt passed to the LLM.
+        """
+        raise NotImplementedError(f"`execute` function of {type(self).__name__} is not implemented!")
+
+    async def async_execute(self, llm: Optional[BaseLLM] = None, inputs: Optional[dict] = None, sys_msg: Optional[str]=None, return_prompt: bool = False, **kwargs) -> Optional[Union[Parser, Tuple[Parser, str]]]:
+        """
+        Asynchronous execution of the action.
+        
+        This method is the asynchronous counterpart of the `execute` method.
+        It allows the action to be executed asynchronously using an LLM.
+        """
+        raise NotImplementedError(f"`async_execute` function of {type(self).__name__} is not implemented!")
+
+class ContextExtraction(Action):
+    """Action for extracting structured inputs from context.
+    
+    This action analyzes a conversation context to extract relevant information
+    that can be used as inputs for other actions. It uses the LLM to interpret
+    unstructured contextual information and format it according to the target
+    action's input requirements.
+    """
+
+    def __init__(self, **kwargs):
+        name = kwargs.pop("name") if "name" in kwargs else CONTEXT_EXTRACTION["name"]
+        description = kwargs.pop("description") if "description" in kwargs else CONTEXT_EXTRACTION["description"]
+        super().__init__(name=name, description=description, **kwargs)
+
+    def get_context_from_messages(self, messages: List[Message]) -> str:
+        str_context = "\n\n".join([str(msg) for msg in messages])
+        return str_context 
+    
+    def execute(self, llm: Optional[BaseLLM] = None, action: Action = None, context: List[Message] = None, **kwargs) -> Union[dict, None]:
+        """Extract structured inputs for an action from conversation context.
+        
+        This method uses the LLM to analyze the conversation context and extract
+        information that matches the input requirements of the target action.
+        
+        Args:
+            llm: The language model to use for extraction.
+            action: The target action whose input requirements (`inputs_format`) define what to extract.
+            context: List of messages providing the conversation context.
+            **kwargs: Additional keyword arguments.
+            
+        Returns:
+            A dictionary containing the extracted inputs for the target action,
+            or None if extraction is not possible (e.g., if the action doesn't
+            require inputs or if context is missing).
+        """
+        if action is None or context is None:
+            return None
+        
+        action_inputs_cls: Type[ActionInput] = action.inputs_format
+        if action_inputs_cls is None:
+            # the action does not require inputs
+            return None
+        
+        action_inputs_desc = action_inputs_cls.get_input_specification()
+        str_context = self.get_context_from_messages(messages=context)
+
+        if not action_inputs_desc or not str_context:
+            return None
+        
+        prompt = CONTEXT_EXTRACTION["prompt"].format(
+            context=str_context,
+            action_name=action.name, 
+            action_description=action.description,
+            action_inputs=action_inputs_desc
+        )
+
+        action_inputs = llm.generate(
+            prompt=prompt, 
+            system_message=CONTEXT_EXTRACTION["system_prompt"],
+            parser=action_inputs_cls
+        )
+        action_inputs_data = action_inputs.get_structured_data()
+
+        return action_inputs_data

evoagentx/actions/agent_generation.py

@@ -0,0 +1,198 @@
+import re
+from pydantic import Field, model_validator
+from typing import Optional, List
+
+from ..core.logging import logger
+from ..core.module import BaseModule
+from ..core.base_config import Parameter
+from ..models.base_model import BaseLLM
+from .action import Action, ActionInput, ActionOutput
+from ..prompts.agent_generator import AGENT_GENERATION_ACTION
+from ..prompts.tool_calling import AGENT_GENERATION_TOOLS_PROMPT
+from ..utils.utils import normalize_text
+
+class AgentGenerationInput(ActionInput):
+    """
+    Input specification for the agent generation action.
+    """
+
+    goal: str = Field(description="A detailed statement of the workflow's goal, explaining the objectives the entire workflow aims to achieve")
+    workflow: str = Field(description="An overview of the entire workflow, detailing all sub-tasks with their respective names, descriptions, inputs, and outputs")
+    task: str = Field(description="A detailed JSON representation of the sub-task requiring agent generation. It should include the task's name, description, inputs, and outputs.")
+
+    history: Optional[str] = Field(default=None, description="Optional field containing previously selected or generated agents.")
+    suggestion: Optional[str] = Field(default=None, description="Optional suggestions to refine the generated agents.")
+    existing_agents: Optional[str] = Field(default=None, description="Optional field containing the description of predefined agents, including each agent's name, role, and available actions.")
+    tools: Optional[str] = Field(default=None, description="Optional field containing the description of tools that agents can use, including each tool's name and functionality.")
+
+
+class GeneratedAgent(BaseModule):
+    """
+    Representation of a generated agent with validation capabilities.
+    """
+
+    name: str 
+    description: str 
+    inputs: List[Parameter]
+    outputs: List[Parameter]
+    prompt: str
+    tool_names: Optional[List[str]] = None
+
+    @classmethod
+    def find_output_name(cls, text: str, outputs: List[str]):
+        def sim(t1: str, t2: str):
+            t1_words = normalize_text(t1).split()
+            t2_words = normalize_text(t2).split()
+            return len(set(t1_words)&set(t2_words))
+        
+        similarities = [sim(text, output) for output in outputs]
+        max_sim = max(similarities)
+        return outputs[similarities.index(max_sim)]
+
+    @model_validator(mode="after")
+    @classmethod
+    def validate_prompt(cls, agent: 'GeneratedAgent'):
+        """Validate and fix the agent's prompt template.
+        
+        This validator ensures that:
+        1. All input parameters are properly referenced in the prompt
+        2. Input references use the correct format with braces
+        3. All output sections match the defined output parameters
+        
+        If there are mismatches in the output sections, it attempts to
+        fix them by finding the most similar output name.
+        
+        Args:
+            agent: The GeneratedAgent instance to validate.
+            
+        Returns:
+            The validated and potentially modified GeneratedAgent.
+            
+        Raises:
+            ValueError: If inputs are missing from the prompt or output sections don't match the defined outputs.
+        """
+        # check whether all the inputs are present in the prompt 
+        input_names = [inp.name for inp in agent.inputs]
+        prompt_has_inputs = [name in agent.prompt for name in input_names]
+        if not all(prompt_has_inputs):
+            missing_input_names = [name for name, has_input in zip(input_names, prompt_has_inputs) if not has_input]
+            raise ValueError(f'The prompt miss inputs: {missing_input_names}')
+        
+        # check the format of the prompt to make sure it is wrapped in brackets. 
+        pattern = r"### Instructions(.*?)### Output Format"
+        prompt = agent.prompt
+
+        def replace_with_braces(match):
+            instructions = match.group(1)
+            for name in input_names:
+                instructions = re.sub(fr'<input>{{*\b{re.escape(name)}\b}}*</input>', fr'<input>{{{name}}}</input>', instructions)
+            return "### Instructions" + instructions + "### Output Format"
+        
+        modified_prompt = re.sub(pattern, replace_with_braces, prompt, flags=re.DOTALL)
+        agent.prompt = modified_prompt
+
+        # check whether all the outputs are present in the prompt
+        prompt = agent.prompt
+        pattern = r"### Output Format(.*)"
+        outputs_names = [out.name for out in agent.outputs]
+
+        def fix_output_names(match):
+            output_format = match.group(1)
+            matches = re.findall(r"## ([^\n#]+)", output_format, flags=re.DOTALL)
+            generated_outputs = [m.strip() for m in matches if m.strip() != "Thought"]
+            # check the number of generated outputs and agent outputs 
+            if len(generated_outputs) != len(outputs_names):
+                raise ValueError(f"The number of outputs in the prompt is different from that defined in the `outputs` field of the agent. The outputs in the prompt are: {generated_outputs}, while the outputs from the agent's `outputs` field are: {outputs_names}")
+            # check whether the generated output names are the same as agent outputs 
+            for generated_output in generated_outputs:
+                if generated_output not in outputs_names:
+                    most_similar_output_name = cls.find_output_name(text=generated_output, outputs=outputs_names)
+                    output_format = output_format.replace(generated_output, most_similar_output_name)
+                    logger.warning(f"Couldn't find output name in prompt ('{generated_output}') in agent's outputs. Replace it with the most similar agent output: '{most_similar_output_name}'")
+            return "### Output Format" + output_format
+        
+        modified_prompt = re.sub(pattern, fix_output_names, prompt, flags=re.DOTALL)
+        agent.prompt = modified_prompt
+
+        return agent
+
+
+class AgentGenerationOutput(ActionOutput):
+
+    selected_agents: List[str] = Field(description="A list of selected agent's names")
+    generated_agents: List[GeneratedAgent] = Field(description="A list of generated agetns to address a sub-task")
+    
+
+class AgentGeneration(Action):
+    """
+    Action for generating agent specifications for workflow tasks.
+    
+    This action analyzes task requirements and generates appropriate agent
+    specifications, including their prompts, inputs, and outputs. It can either
+    select from existing agents or create new ones tailored to the task.
+    """
+
+    def __init__(self, **kwargs):
+        name = kwargs.pop("name") if "name" in kwargs else AGENT_GENERATION_ACTION["name"]
+        description = kwargs.pop("description") if "description" in kwargs else AGENT_GENERATION_ACTION["description"]
+        prompt = kwargs.pop("prompt") if "prompt" in kwargs else AGENT_GENERATION_ACTION["prompt"]
+        # inputs_format = kwargs.pop("inputs_format") if "inputs_format" in kwargs else AgentGenerationInput
+        # outputs_format = kwargs.pop("outputs_format") if "outputs_format" in kwargs else AgentGenerationOutput
+        inputs_format = kwargs.pop("inputs_format", None) or AgentGenerationInput
+        outputs_format = kwargs.pop("outputs_format", None) or AgentGenerationOutput 
+        tools = kwargs.pop("tools", None)
+        super().__init__(name=name, description=description, prompt=prompt, inputs_format=inputs_format, outputs_format=outputs_format, **kwargs)
+        self.tools = tools
+    
+    def execute(self, llm: Optional[BaseLLM] = None, inputs: Optional[dict] = None, sys_msg: Optional[str]=None, return_prompt: bool = False, **kwargs) -> AgentGenerationOutput:
+        """Execute the agent generation process.
+        
+        This method uses the provided language model to generate agent specifications
+        based on the workflow context and task requirements.
+        
+        Args:
+            llm: The language model to use for generation.
+            inputs: Input data containing workflow and task information.
+            sys_msg: Optional system message for the language model.
+            return_prompt: Whether to return both the generated agents and the prompt used.
+            **kwargs: Additional keyword arguments.
+            
+        Returns:
+            If return_prompt is False (default): The generated agents output.
+            If return_prompt is True: A tuple of (generated agents, prompt used).
+            
+        Raises:
+            ValueError: If the inputs are None or empty.
+        """
+        if not inputs:
+            logger.error("AgentGeneration action received invalid `inputs`: None or empty.")
+            raise ValueError('The `inputs` to AgentGeneration action is None or empty.')
+        
+        inputs_format: AgentGenerationInput = self.inputs_format
+        outputs_format: AgentGenerationOutput = self.outputs_format
+
+        prompt_params_names = inputs_format.get_attrs()
+        prompt_params_values = {param: inputs.get(param, "") for param in prompt_params_names}
+        if self.tools:
+            tool_description = [
+                {
+                    tool.name: [
+                        s["function"]["description"] for s in tool.get_tool_schemas()
+                    ],
+                }
+                for tool in self.tools
+            ]
+            prompt_params_values["tools"] = AGENT_GENERATION_TOOLS_PROMPT.format(tools_description=tool_description)
+        prompt = self.prompt.format(**prompt_params_values)
+        agents = llm.generate(
+            prompt = prompt, 
+            system_message = sys_msg, 
+            parser=outputs_format,
+            parse_mode="json"
+        )
+        
+        if return_prompt:
+            return agents, prompt
+        
+        return agents
+    

evoagentx/actions/code_extraction.py

@@ -0,0 +1,276 @@
+import os
+from typing import Optional, List, Dict
+from pydantic import Field
+
+from ..models.base_model import BaseLLM, LLMOutputParser
+from .action import Action, ActionInput, ActionOutput
+from ..prompts.code_extraction import CODE_EXTRACTION
+
+
+class CodeExtractionInput(ActionInput):
+    """
+    Input parameters for the CodeExtraction action.
+    """
+    code_string: str = Field(description="The string containing code blocks to extract")
+    target_directory: str = Field(description="The directory path where extracted code files will be saved")
+    project_name: Optional[str] = Field(default=None, description="Optional name for the project folder")
+
+
+class CodeExtractionOutput(ActionOutput):
+    """
+    Output of the CodeExtraction action.
+    """
+    extracted_files: Dict[str, str] = Field(description="Map of filename to file path of saved files")
+    main_file: Optional[str] = Field(default=None, description="Path to the main file if identified")
+    error: Optional[str] = Field(default=None, description="Error message if any operation failed")
+
+
+class CodeBlockInfo(LLMOutputParser):
+    """
+    Information about an extracted code block.
+    """
+    language: str = Field(description="Programming language of the code block")
+    filename: str = Field(description="Suggested filename for the code block")
+    content: str = Field(description="The actual code content")
+
+
+class CodeBlockList(LLMOutputParser):
+    """
+    List of code blocks extracted from text.
+    """
+    code_blocks: List[CodeBlockInfo] = Field(description="List of code blocks")
+
+
+class CodeExtraction(Action):
+    """
+    An action that extracts and organizes code blocks from text.
+    
+    This action uses an LLM to analyze text containing code blocks, extract them,
+    suggest appropriate filenames, and save them to a specified directory. It can
+    also identify which file is likely the main entry point based on heuristics.
+    
+    Attributes:
+        name: The name of the action.
+        description: A description of what the action does.
+        prompt: The prompt template used by the action.
+        inputs_format: The expected format of inputs to this action.
+        outputs_format: The format of the action's output.
+    """
+
+    def __init__(self, **kwargs):
+        
+        name = kwargs.pop("name") if "name" in kwargs else CODE_EXTRACTION["name"]
+        description = kwargs.pop("description") if "description" in kwargs else CODE_EXTRACTION["description"]
+        prompt = kwargs.pop("prompt") if "prompt" in kwargs else CODE_EXTRACTION["prompt"]
+        # inputs_format = kwargs.pop("inputs_format") if "inputs_format" in kwargs else CodeExtractionInput
+        # outputs_format = kwargs.pop("outputs_format") if "outputs_format" in kwargs else CodeExtractionOutput
+        inputs_format = kwargs.pop("inputs_format", None) or CodeExtractionInput
+        outputs_format = kwargs.pop("outputs_format", None) or CodeExtractionOutput
+        super().__init__(name=name, description=description, prompt=prompt, inputs_format=inputs_format, outputs_format=outputs_format, **kwargs)
+
+    def identify_main_file(self, saved_files: Dict[str, str]) -> Optional[str]:
+        """Identify the main file from the saved files based on content and file type.
+        
+        This method uses a combination of common filename conventions and content
+        analysis to determine which file is likely the main entry point of a project.
+        
+        Args:
+            saved_files: Dictionary mapping filenames to their full paths
+            
+        Returns:
+            Path to the main file if found, None otherwise
+            
+        """
+        # Priority lookup for common main files by language
+        main_file_priorities = [
+            # HTML files
+            "index.html",
+            # Python files
+            "main.py", 
+            "app.py",
+            # JavaScript files
+            "index.js",
+            "main.js",
+            "app.js",
+            # Java files
+            "Main.java",
+            # C/C++ files
+            "main.cpp", 
+            "main.c",
+            # Go files
+            "main.go",
+            # Other common entry points
+            "index.php",
+            "Program.cs"
+        ]
+        
+        # First check priority list
+        for main_file in main_file_priorities:
+            if main_file in saved_files:
+                return saved_files[main_file]
+        
+        # If no priority file found, use heuristics based on file extensions
+        
+        # If we have HTML files, use the first one
+        html_files = {k: v for k, v in saved_files.items() if k.endswith('.html')}
+        if html_files:
+            return next(iter(html_files.values()))
+        
+        # Check for Python files with "__main__" section
+        py_files = {k: v for k, v in saved_files.items() if k.endswith('.py')}
+        if py_files:
+            for filename, path in py_files.items():
+                with open(path, 'r', encoding='utf-8') as f:
+                    content = f.read()
+                    if "if __name__ == '__main__'" in content or 'if __name__ == "__main__"' in content:
+                        return path
+            # If no main found, return the first Python file
+            if py_files:
+                return next(iter(py_files.values()))
+        
+        # If we have Java files, look for one with a main method
+        java_files = {k: v for k, v in saved_files.items() if k.endswith('.java')}
+        if java_files:
+            for filename, path in java_files.items():
+                with open(path, 'r', encoding='utf-8') as f:
+                    content = f.read()
+                    if "public static void main" in content:
+                        return path
+            # If no main found, return the first Java file
+            if java_files:
+                return next(iter(java_files.values()))
+                
+        # For JavaScript applications
+        js_files = {k: v for k, v in saved_files.items() if k.endswith('.js')}
+        if js_files:
+            return next(iter(js_files.values()))
+                
+        # If all else fails, return the first file
+        if saved_files:
+            return next(iter(saved_files.values()))
+                
+        # No files found
+        return None
+
+    def save_code_blocks(self, code_blocks: List[Dict], target_directory: str) -> Dict[str, str]:
+        """Save code blocks to files in the target directory.
+        
+        Creates the target directory if it doesn't exist and saves each code block
+        to a file with an appropriate name, handling filename conflicts.
+        
+        Args:
+            code_blocks: List of dictionaries containing code block information
+            target_directory: Directory path where files should be saved
+            
+        Returns:
+            Dictionary mapping filenames to their full paths
+        """
+        os.makedirs(target_directory, exist_ok=True)
+        saved_files = {}
+        
+        for block in code_blocks:
+            filename = block.get("filename", "unknown.txt")
+            content = block.get("content", "")
+            
+            # Skip empty blocks
+            if not content.strip():
+                continue
+            
+            # Handle filename conflicts
+            base_filename = filename
+            counter = 1
+            while filename in saved_files:
+                name_parts = base_filename.split('.')
+                if len(name_parts) > 1:
+                    filename = f"{'.'.join(name_parts[:-1])}_{counter}.{name_parts[-1]}"
+                else:
+                    filename = f"{base_filename}_{counter}"
+                counter += 1
+                
+            # Save to file
+            file_path = os.path.join(target_directory, filename)
+            with open(file_path, 'w', encoding='utf-8') as f:
+                f.write(content)
+                
+            # Add to map
+            saved_files[filename] = file_path
+            
+        return saved_files
+
+    def execute(self, llm: Optional[BaseLLM] = None, inputs: Optional[dict] = None, sys_msg: Optional[str]=None, return_prompt: bool = False, **kwargs) -> CodeExtractionOutput:
+        """Execute the CodeExtraction action.
+        
+        Extracts code blocks from the provided text using the specified LLM,
+        saves them to the target directory, and identifies the main file.
+        
+        Args:
+            llm: The LLM to use for code extraction
+            inputs: Dictionary containing:
+                - code_string: The string with code blocks to extract
+                - target_directory: Where to save the files
+                - project_name: Optional project folder name
+            sys_msg: Optional system message override for the LLM
+            return_prompt: Whether to return the prompt along with the result
+            **kwargs (Any): Additional keyword arguments
+            
+        Returns:
+            CodeExtractionOutput with extracted file information
+        """
+        if not llm:
+            error_msg = "CodeExtraction action requires an LLM."
+            return CodeExtractionOutput(extracted_files={}, error=error_msg)
+            
+        if not inputs:
+            error_msg = "CodeExtraction action received invalid `inputs`: None or empty."
+            return CodeExtractionOutput(extracted_files={}, error=error_msg)
+        
+        code_string = inputs.get("code_string", "")
+        target_directory = inputs.get("target_directory", "")
+        project_name = inputs.get("project_name", None)
+        
+        if not code_string:
+            error_msg = "No code string provided."
+            return CodeExtractionOutput(extracted_files={}, error=error_msg)
+            
+        if not target_directory:
+            error_msg = "No target directory provided."
+            return CodeExtractionOutput(extracted_files={}, error=error_msg)
+        
+        # Create project folder if name is provided
+        if project_name:
+            project_dir = os.path.join(target_directory, project_name)
+        else:
+            project_dir = target_directory
+            
+        try:
+            # Use LLM to extract code blocks and suggest filenames
+            prompt_params = {"code_string": code_string}
+            system_message = CODE_EXTRACTION["system_prompt"] if sys_msg is None else sys_msg
+
+            llm_response: CodeBlockList = llm.generate(
+                prompt=self.prompt.format(**prompt_params),
+                system_message=system_message,
+                parser=CodeBlockList,
+                parse_mode="json"
+            )
+            code_blocks = llm_response.get_structured_data().get("code_blocks", [])
+
+            # Save code blocks to files
+            saved_files = self.save_code_blocks(code_blocks, project_dir)
+            
+            # Identify main file
+            main_file = self.identify_main_file(saved_files)
+            
+            result = CodeExtractionOutput(
+                extracted_files=saved_files,
+                main_file=main_file
+            )
+            
+            if return_prompt:
+                return result, self.prompt.format(**prompt_params)
+                
+            return result
+            
+        except Exception as e:
+            error_msg = f"Error extracting code: {str(e)}"
+            return CodeExtractionOutput(extracted_files={}, error=error_msg)

evoagentx/actions/code_verification.py

@@ -0,0 +1,63 @@
+from pydantic import Field
+from typing import Optional
+
+from ..core.logging import logger
+from ..core.module_utils import extract_code_blocks
+from ..models.base_model import BaseLLM
+from .action import Action, ActionInput, ActionOutput
+from ..prompts.code_verification import CODE_VERIFICATION_ACTION
+
+
+class CodeVerificationInput(ActionInput):
+
+    code: str = Field(description="The code string to be verified for correctness and completeness.")
+    requirements: Optional[str] = Field(default=None, description="Optional field containing requirements or specifications for the code.")
+
+
+class CodeVerificationOutput(ActionOutput):
+
+    analysis_summary: Optional[str] = Field(default=None, description="Brief summary of your findings, highlighting key issues or confirming overall quality.")
+    issues_identified: Optional[str] = Field(default=None, description="Categorized list of issues found, with explanation of impact and severity.")
+    thought_process: Optional[str] = Field(default=None, description="Detailed explanation of your verification reasoning and methodology applied.")
+    modification_strategy: Optional[str] = Field(default=None, description="Describe the changes you made (or will make) to address the issues. Include any assumptions, design choices, or additional components you decided to add to make the code complete and robust.")
+    verified_code: str = Field(description="The complete, corrected code if issues are found, or the original code if no issues are found.")
+
+
+class CodeVerification(Action):
+
+    def __init__(self, **kwargs):
+
+        name = kwargs.pop("name") if "name" in kwargs else CODE_VERIFICATION_ACTION["name"]
+        description = kwargs.pop("description") if "description" in kwargs else CODE_VERIFICATION_ACTION["description"]
+        prompt = kwargs.pop("prompt") if "prompt" in kwargs else CODE_VERIFICATION_ACTION["prompt"]
+        # inputs_format = kwargs.pop("inputs_format") if "inputs_format" in kwargs else CodeVerificationInput
+        # outputs_format = kwargs.pop("outputs_format") if "outputs_format" in kwargs else CodeVerificationOutput
+        inputs_format = kwargs.pop("inputs_format", None) or CodeVerificationInput
+        outputs_format = kwargs.pop("outputs_format", None) or CodeVerificationOutput
+        super().__init__(name=name, description=description, prompt=prompt, inputs_format=inputs_format, outputs_format=outputs_format, **kwargs)
+    
+    def execute(self, llm: Optional[BaseLLM] = None, inputs: Optional[dict] = None, sys_msg: Optional[str]=None, return_prompt: bool = False, **kwargs) -> CodeVerificationOutput:
+
+        if not inputs:
+            logger.error("CodeVerification action received invalid `inputs`: None or empty.")
+            raise ValueError('The `inputs` to CodeVerification action is None or empty.')
+
+        prompt_params_names = ["code", "requirements"]
+        prompt_params_values = {param: inputs.get(param, "Not Provided") for param in prompt_params_names}
+        prompt = self.prompt.format(**prompt_params_values)
+        response = llm.generate(prompt = prompt, system_message=sys_msg)
+        
+        try:
+            verification_result = self.outputs_format.parse(response.content, parse_mode="title")
+        except Exception:
+            try:
+                code_blocks = extract_code_blocks(response.content, return_type=True)
+                code = "\n\n".join([f"```{code_type}\n{code}\n```" for code_type, code in code_blocks])
+                verification_result = self.outputs_format(verified_code=code)
+            except Exception:
+                raise ValueError(f"Failed to extract code blocks from the response: {response.content}")
+        
+        if return_prompt:
+            return verification_result, prompt
+        
+        return verification_result

evoagentx/actions/customize_action.py

@@ -0,0 +1,559 @@
+from pydantic import Field
+from typing import Optional, Any, Callable, List, Union
+import re
+import json
+import asyncio
+import inspect
+import concurrent.futures
+
+from ..core.logging import logger
+from ..models.base_model import BaseLLM
+from .action import Action
+from ..core.message import Message
+from ..prompts.template import StringTemplate, ChatTemplate
+from ..prompts.tool_calling import OUTPUT_EXTRACTION_PROMPT, TOOL_CALLING_TEMPLATE, TOOL_CALLING_HISTORY_PROMPT, TOOL_CALLING_RETRY_PROMPT
+from ..tools.tool import Toolkit
+from ..core.registry import MODULE_REGISTRY
+from ..models.base_model import LLMOutputParser
+from ..core.module_utils import parse_json_from_llm_output, parse_json_from_text
+
+class CustomizeAction(Action):
+
+    parse_mode: Optional[str] = Field(default="title", description="the parse mode of the action, must be one of: ['title', 'str', 'json', 'xml', 'custom']")
+    parse_func: Optional[Callable] = Field(default=None, exclude=True, description="the function to parse the LLM output. It receives the LLM output and returns a dict.")
+    title_format: Optional[str] = Field(default="## {title}", exclude=True, description="the format of the title. It is used when the `parse_mode` is 'title'.")
+    custom_output_format: Optional[str] = Field(default=None, exclude=True, description="the format of the output. It is used when the `prompt_template` is provided.")
+
+    tools: Optional[List[Toolkit]] = Field(default=None, description="The tools that the action can use")
+    conversation: Optional[Message] = Field(default=None, description="Current conversation state")
+
+    max_tool_try: int = Field(default=2, description="Maximum number of tool calling attempts allowed")
+    
+    def __init__(self, **kwargs):
+
+        name = kwargs.pop("name", "CustomizeAction")
+        description = kwargs.pop("description", "Customized action that can use tools to accomplish its task")
+
+        super().__init__(name=name, description=description, **kwargs)
+        
+        # Validate that at least one of prompt or prompt_template is provided
+        if not self.prompt and not self.prompt_template:
+            raise ValueError("`prompt` or `prompt_template` is required when creating CustomizeAction action")
+        # Prioritize template and give warning if both are provided
+        if self.prompt and self.prompt_template:
+            logger.warning("Both `prompt` and `prompt_template` are provided for CustomizeAction action. Prioritizing `prompt_template` and ignoring `prompt`.")
+        if self.tools:
+            self.tools_caller = {}
+            self.add_tools(self.tools)
+    
+    def prepare_action_prompt(
+        self, 
+        inputs: Optional[dict] = None, 
+        system_prompt: Optional[str] = None, 
+        **kwargs
+    ) -> Union[str, List[dict]]:
+        """Prepare prompt for action execution.
+        
+        This helper function transforms the input dictionary into a formatted prompt
+        for the language model, handling different prompting modes.
+        
+        Args:
+            inputs: Dictionary of input parameters
+            system_prompt: Optional system prompt to include
+            
+        Returns:
+            Union[str, List[dict]]: Formatted prompt ready for LLM (string or chat messages)
+            
+        Raises:
+            TypeError: If an input value type is not supported
+            ValueError: If neither prompt nor prompt_template is available
+        """
+        # Process inputs into prompt parameter values
+        if inputs is None:
+            inputs = {}
+            
+        prompt_params_names = self.inputs_format.get_attrs()
+        prompt_params_values = {}
+        for param in prompt_params_names:
+            value = inputs.get(param, "")
+            if isinstance(value, str):
+                prompt_params_values[param] = value
+            elif isinstance(value, (dict, list)):
+                prompt_params_values[param] = json.dumps(value, indent=4)
+            else:
+                raise TypeError(f"The input type {type(value)} is invalid! Valid types: [str, dict, list].")
+        
+        if self.prompt:
+            prompt = self.prompt.format(**prompt_params_values) if prompt_params_values else self.prompt
+            if self.tools:
+                tools_schemas = [j["function"] for i in [tool.get_tool_schemas() for tool in self.tools] for j in i]
+                prompt += "\n\n" + TOOL_CALLING_TEMPLATE.format(tools_description = tools_schemas)
+            return prompt
+        else:
+            # Use goal-based tool calling mode
+            if self.tools:
+                self.prompt_template.set_tools(self.tools)
+            return self.prompt_template.format(
+                system_prompt=system_prompt,
+                values=prompt_params_values,
+                inputs_format=self.inputs_format,
+                outputs_format=self.outputs_format,
+                parse_mode=self.parse_mode,
+                title_format=self.title_format,
+                custom_output_format=self.custom_output_format,
+                tools=self.tools
+            )
+
+    def prepare_extraction_prompt(self, llm_output_content: str) -> str:
+        """Prepare extraction prompt for fallback extraction when parsing fails.
+        
+        Args:
+            self: The action instance
+            llm_output_content: Raw output content from LLM
+            
+        Returns:
+            str: Formatted extraction prompt
+        """
+        attr_descriptions: dict = self.outputs_format.get_attr_descriptions()
+        output_description_list = [] 
+        for i, (name, desc) in enumerate(attr_descriptions.items()):
+            output_description_list.append(f"{i+1}. {name}\nDescription: {desc}")
+        output_description = "\n\n".join(output_description_list)
+        return OUTPUT_EXTRACTION_PROMPT.format(text=llm_output_content, output_description=output_description)
+    
+    def _get_unique_class_name(self, candidate_name: str) -> str:
+        """
+        Get a unique class name by checking if it already exists in the registry.
+        If it does, append "Vx" to make it unique.
+        """
+        if not MODULE_REGISTRY.has_module(candidate_name):
+            return candidate_name 
+        
+        i = 1 
+        while True:
+            unique_name = f"{candidate_name}V{i}"
+            if not MODULE_REGISTRY.has_module(unique_name):
+                break
+            i += 1 
+        return unique_name 
+    
+    def add_tools(self, tools: Union[Toolkit, List[Toolkit]]):
+        if not tools:
+            return
+        if isinstance(tools,Toolkit):
+            tools = [tools]
+        if not all(isinstance(tool, Toolkit) for tool in tools):
+            raise TypeError("`tools` must be a Toolkit or list of Toolkit instances.")
+        if not self.tools:
+            self.tools_caller = {}
+            self.tools = []
+        # self.tools += tools
+        # tools_callers = [tool.get_tools() for tool in tools]
+        # tools_callers = [j for i in tools_callers for j in i]
+        # for tool_caller in tools_callers:
+        #     self.tools_caller[tool_caller.name] = tool_caller
+
+        # avoid duplication & type checks 
+        for toolkit in tools:
+            try:
+                tool_callers = toolkit.get_tools()
+                if not isinstance(tool_callers, list):
+                    logger.warning(f"Expected list of tool functions from '{toolkit.name}.get_tools()', got {type(tool_callers)}.")
+                    continue 
+                
+                # add tool callers to the tools_caller dictionary
+                valid_tools_count = 0 
+                valid_tools_names, valid_tool_callers = [], []
+                for tool_caller in tool_callers:
+                    tool_caller_name = getattr(tool_caller, "name", None)
+                    if not tool_caller_name or not callable(tool_caller):
+                        logger.warning(f"Invalid tool function in '{toolkit.name}': missing name or not callable.")
+                        continue
+                    if tool_caller_name in self.tools_caller:
+                        logger.warning(f"Duplicate tool function '{tool_caller_name}' detected. Overwriting previous function.")
+                    # self.tools_caller[tool_caller_name] = tool_caller
+                    valid_tools_count += 1
+                    valid_tools_names.append(tool_caller_name)
+                    valid_tool_callers.append(tool_caller)
+
+                if valid_tools_count == 0:
+                    logger.info(f"No valid tools found in toolkit '{toolkit.name}'. Skipping.")
+                    continue 
+
+                if valid_tools_count > 0 and all(name in self.tools_caller for name in valid_tools_names):
+                    logger.info(f"All tools from toolkit '{toolkit.name}' are already added. Skipping.")
+                    continue
+                
+                if valid_tools_count > 0:
+                    self.tools_caller.update({name: caller for name, caller in zip(valid_tools_names, valid_tool_callers)}) 
+                
+                # only add toolkit if at least one valid tool is added and toolkit is not already added 
+                existing_toolkit_names = {tkt.name for tkt in self.tools}
+                if valid_tools_count > 0 and toolkit.name not in existing_toolkit_names:
+                    self.tools.append(toolkit)
+                if valid_tools_count > 0:
+                    logger.info(f"Added toolkit '{toolkit.name}' with {valid_tools_count} valid tools in {self.name}: {valid_tools_names}.")
+            
+            except Exception as e:
+                logger.error(f"Failed to load tools from toolkit '{toolkit.name}': {e}")
+    
+    
+    def _extract_tool_calls(self, llm_output: str, llm: Optional[BaseLLM] = None) -> List[dict]:
+        pattern = r"<ToolCalling>\s*(.*?)\s*</ToolCalling>"
+    
+        
+        # Find all ToolCalling blocks in the output
+        matches = re.findall(pattern, llm_output, re.DOTALL)
+
+        if not matches:
+            return []
+        
+        parsed_tool_calls = []
+        for match_content in matches:
+            try:
+                json_content = match_content.strip()
+                json_list = parse_json_from_text(json_content)
+                if not json_list:
+                    logger.warning("No valid JSON found in ToolCalling block")
+                    continue
+                # Only use the first JSON string from each block
+                parsed_tool_call = json.loads(json_list[0])
+                if isinstance(parsed_tool_call, dict):
+                    parsed_tool_calls.append(parsed_tool_call)
+                elif isinstance(parsed_tool_call, list):
+                    parsed_tool_calls.extend(parsed_tool_call)
+                else:
+                    logger.warning(f"Invalid tool call format: {parsed_tool_call}")
+                    continue
+            except (json.JSONDecodeError, IndexError) as e:
+                logger.warning(f"Failed to parse tool calls from LLM output: {e}")
+                if llm is not None:
+                    retry_prompt = TOOL_CALLING_RETRY_PROMPT.format(text=match_content)
+                    try:
+                        fixed_output = llm.generate(prompt=retry_prompt).content.strip()
+                        logger.info(f"Retrying tool call parse with fixed output:\n{fixed_output}")
+                        
+                        fixed_list = parse_json_from_text(fixed_output)
+                        if fixed_list:
+                            parsed_tool_call = json.loads(fixed_list[0])
+                            if isinstance(parsed_tool_call, dict):
+                                parsed_tool_calls.append(parsed_tool_call)
+                        elif isinstance(parsed_tool_call, list):
+                            parsed_tool_calls.extend(parsed_tool_call)
+                    except Exception as retry_err:
+                        logger.error(f"Retry failed: {retry_err}")
+                        continue
+            else:
+                continue
+
+        return parsed_tool_calls
+    
+    def _extract_output(self, llm_output: Any, llm: BaseLLM = None, **kwargs):
+
+        # Get the raw output content
+        llm_output_content = getattr(llm_output, "content", str(llm_output))
+        
+        # Check if there are any defined output fields
+        output_attrs = self.outputs_format.get_attrs()
+        
+        # If no output fields are defined, create a simple content-only output
+        if not output_attrs:
+            # Create output with just the content field
+            output = self.outputs_format.parse(content=llm_output_content)
+            # print("Created simple content output for agent with no defined outputs:")
+            # print(output)
+            return output
+        
+        # Use the action's parse_mode and parse_func for parsing
+        try:
+            # Use the outputs_format's parse method with the action's parse settings
+            parsed_output = self.outputs_format.parse(
+                content=llm_output_content,
+                parse_mode=self.parse_mode,
+                parse_func=getattr(self, 'parse_func', None),
+                title_format=getattr(self, 'title_format', "## {title}")
+            )
+            
+            # print("Successfully parsed output using action's parse settings:")
+            # print(parsed_output)
+            return parsed_output
+            
+        except Exception as e:
+            logger.info(f"Failed to parse with action's parse settings: {e}")
+            logger.info("Falling back to using LLM to extract outputs...")
+            
+            # Fall back to extraction prompt if direct parsing fails
+            extraction_prompt = self.prepare_extraction_prompt(llm_output_content)
+                
+            llm_extracted_output: LLMOutputParser = llm.generate(prompt=extraction_prompt)
+            llm_extracted_data: dict = parse_json_from_llm_output(llm_extracted_output.content)
+            output = self.outputs_format.from_dict(llm_extracted_data)
+            
+            # print("Extracted output using fallback:")
+            # print(output)
+            return output
+    
+    async def _async_extract_output(self, llm_output: Any, llm: BaseLLM = None, **kwargs):
+        
+        # Get the raw output content
+        llm_output_content = getattr(llm_output, "content", str(llm_output))
+        
+        # Check if there are any defined output fields
+        output_attrs = self.outputs_format.get_attrs()
+        
+        # If no output fields are defined, create a simple content-only output
+        if not output_attrs:
+            # Create output with just the content field
+            output = self.outputs_format.parse(content=llm_output_content)
+            # print("Created simple content output for agent with no defined outputs:")
+            # print(output)
+            return output
+        
+        # Use the action's parse_mode and parse_func for parsing
+        try:
+            # Use the outputs_format's parse method with the action's parse settings
+            parsed_output = self.outputs_format.parse(
+                content=llm_output_content,
+                parse_mode=self.parse_mode,
+                parse_func=getattr(self, 'parse_func', None),
+                title_format=getattr(self, 'title_format', "## {title}")
+            )
+            
+            # print("Successfully parsed output using action's parse settings:")
+            # print(parsed_output)
+            return parsed_output
+            
+        except Exception as e:
+            logger.info(f"Failed to parse with action's parse settings: {e}")
+            logger.info("Falling back to using LLM to extract outputs...")
+            
+            # Fall back to extraction prompt if direct parsing fails
+            extraction_prompt = self.prepare_extraction_prompt(llm_output_content)
+                
+            llm_extracted_output = await llm.async_generate(prompt=extraction_prompt)
+            llm_extracted_data: dict = parse_json_from_llm_output(llm_extracted_output.content)
+            output = self.outputs_format.from_dict(llm_extracted_data)
+            
+            # print("Extracted output using fallback:")
+            # print(output)
+            return output
+    
+    def _call_single_tool(self, function_param: dict) -> tuple:
+        try:
+            function_name = function_param.get("function_name")
+            function_args = function_param.get("function_args") or {}
+    
+            if not function_name:
+                return None, "No function name provided"
+
+            callable_fn = self.tools_caller.get(function_name)
+            if not callable(callable_fn):
+                return None, f"Function '{function_name}' not found or not callable"
+            
+            print("_____________________ Start Function Calling _____________________")
+            print(f"Executing function calling: {function_name} with parameters: {function_args}")
+            result = callable_fn(**function_args)
+            return result, None
+
+        except Exception as e:
+            logger.error(f"Error executing tool {function_name}: {e}")
+            return None, f"Error executing tool {function_name}: {str(e)}"
+
+    def _calling_tools(self, tool_call_args: List[dict]) -> dict:
+        ## ___________ Call the tools in parallel___________
+        errors = []
+        results = []
+        
+        with concurrent.futures.ThreadPoolExecutor() as executor:
+            future_to_tool = {executor.submit(self._call_single_tool, param): param for param in tool_call_args}
+            
+            for future in concurrent.futures.as_completed(future_to_tool):
+                result, error = future.result()
+                if error:
+                    errors.append(error)
+                if result is not None:
+                    results.append(result)
+
+        return {"result": results, "error": errors}
+
+    async def _async_call_single_tool(self, function_param: dict) -> tuple:
+        try:
+            function_name = function_param.get("function_name")
+            function_args = function_param.get("function_args") or {}
+
+            if not function_name:
+                return None, "No function name provided"
+
+            callable_fn = self.tools_caller.get(function_name)
+            if not callable(callable_fn):
+                return None, f"Function '{function_name}' not found or not callable"
+
+            print("_____________________ Start Function Calling _____________________")
+            print(f"Executing function calling: {function_name} with parameters: {function_args}")
+
+            if inspect.iscoroutinefunction(callable_fn):
+                result = await callable_fn(**function_args)
+            else:
+                loop = asyncio.get_running_loop()
+                result = await loop.run_in_executor(None, lambda: callable_fn(**function_args))
+            
+            return result, None
+        
+        except Exception as e:
+            logger.error(f"Error executing tool {function_name}: {e}")
+            return None, f"Error executing tool {function_name}: {str(e)}"
+
+    async def _async_calling_tools(self, tool_call_args: List[dict]) -> dict:
+        ## ___________ Call the tools concurrently ___________
+        tasks = [self._async_call_single_tool(param) for param in tool_call_args]
+        results_with_errors = await asyncio.gather(*tasks)
+
+        results = [res for res, err in results_with_errors if err is None and res is not None]
+        errors = [err for _, err in results_with_errors if err is not None]
+
+        return {"result": results, "error": errors}
+    
+    def execute(self, llm: Optional[BaseLLM] = None, inputs: Optional[dict] = None, sys_msg: Optional[str]=None, return_prompt: bool = False, time_out = 0, **kwargs):
+        # Allow empty inputs if the action has no required input attributes
+        input_attributes: dict = self.inputs_format.get_attr_descriptions()
+        if not inputs and input_attributes:
+            logger.error("CustomizeAction action received invalid `inputs`: None or empty.")
+            raise ValueError('The `inputs` to CustomizeAction action is None or empty.')
+        # Set inputs to empty dict if None and no inputs are required
+        if inputs is None:
+            inputs = {}
+        final_llm_response = None
+        
+        if self.prompt_template:
+
+            if isinstance(self.prompt_template, ChatTemplate):
+                # must determine whether prompt_template is ChatTemplate first since ChatTemplate is a subclass of StringTemplate
+                conversation = self.prepare_action_prompt(inputs=inputs, system_prompt=sys_msg)
+            elif isinstance(self.prompt_template, StringTemplate):
+                conversation = [{"role": "system", "content": self.prepare_action_prompt(inputs=inputs, system_prompt=sys_msg)}]
+            else:
+                raise ValueError(f"`prompt_template` must be a StringTemplate or ChatTemplate instance, but got {type(self.prompt_template)}")
+        else:
+            conversation = [{"role": "system", "content": sys_msg}, {"role": "user", "content": self.prepare_action_prompt(inputs=inputs, system_prompt=sys_msg)}]
+        
+        
+        ## 1. get all the input parameters
+        prompt_params_values = {k: inputs.get(k, "") for k in input_attributes.keys()}
+        while True:
+            ### Generate response from LLM
+            if time_out > self.max_tool_try:
+                # Get the appropriate prompt for return
+                current_prompt = self.prepare_action_prompt(inputs=prompt_params_values or {})
+                # Use the final LLM response if available, otherwise fall back to execution history
+                content_to_extract = final_llm_response if final_llm_response is not None else "{content}".format(content = conversation)
+                if return_prompt:
+                    return self._extract_output(content_to_extract, llm = llm), current_prompt
+                return self._extract_output(content_to_extract, llm = llm) 
+            time_out += 1
+            
+            # Handle both string prompts and chat message lists
+            llm_response = llm.generate(messages=conversation)
+            conversation.append({"role": "assistant", "content": llm_response.content})
+            
+            # Store the final LLM response
+            final_llm_response = llm_response
+            
+            tool_call_args = self._extract_tool_calls(llm_response.content)
+            if not tool_call_args:
+                break
+            
+            logger.info("Extracted tool call args:")
+            logger.info(json.dumps(tool_call_args, indent=4))
+            
+            results = self._calling_tools(tool_call_args)
+            
+            logger.info("Tool call results:")
+            logger.info(json.dumps(results, indent=4))
+            
+            conversation.append({"role": "assistant", "content": TOOL_CALLING_HISTORY_PROMPT.format(
+                iteration_number=time_out,
+                tool_call_args=f"{tool_call_args}",
+                results=f"{results}"
+            )})
+        
+        # Get the appropriate prompt for return
+        current_prompt = self.prepare_action_prompt(inputs=prompt_params_values or {})
+        # Use the final LLM response if available, otherwise fall back to execution history
+        content_to_extract = final_llm_response if final_llm_response is not None else "{content}".format(content = conversation)
+        if return_prompt:
+            return self._extract_output(content_to_extract, llm = llm), current_prompt
+        return self._extract_output(content_to_extract, llm = llm)
+        
+
+    async def async_execute(self, llm: Optional[BaseLLM] = None, inputs: Optional[dict] = None, sys_msg: Optional[str]=None, return_prompt: bool = False, time_out = 0, **kwargs):
+        # Allow empty inputs if the action has no required input attributes
+        input_attributes: dict = self.inputs_format.get_attr_descriptions()
+        if not inputs and input_attributes:
+            logger.error("CustomizeAction action received invalid `inputs`: None or empty.")
+            raise ValueError('The `inputs` to CustomizeAction action is None or empty.')
+        # Set inputs to empty dict if None and no inputs are required
+        if inputs is None:
+            inputs = {}
+        final_llm_response = None
+        
+        if self.prompt_template:
+            if isinstance(self.prompt_template, ChatTemplate):
+                # must determine whether prompt_template is ChatTemplate first since ChatTemplate is a subclass of StringTemplate
+                conversation = self.prepare_action_prompt(inputs=inputs, system_prompt=sys_msg)
+            elif isinstance(self.prompt_template, StringTemplate):
+                conversation = [{"role": "system", "content": self.prepare_action_prompt(inputs=inputs, system_prompt=sys_msg)}]
+            else:
+                raise ValueError(f"`prompt_template` must be a StringTemplate or ChatTemplate instance, but got {type(self.prompt_template)}")
+        else:
+            conversation = [{"role": "system", "content": sys_msg}, {"role": "user", "content": self.prepare_action_prompt(inputs=inputs, system_prompt=sys_msg)}]
+        
+        
+        ## 1. get all the input parameters
+        prompt_params_values = {k: inputs.get(k, "") for k in input_attributes.keys()}
+        while True:
+            ### Generate response from LLM
+            if time_out > self.max_tool_try:
+                # Get the appropriate prompt for return
+                current_prompt = self.prepare_action_prompt(inputs=prompt_params_values or {})
+                # Use the final LLM response if available, otherwise fall back to execution history
+                content_to_extract = final_llm_response if final_llm_response is not None else "{content}".format(content = conversation)
+                if return_prompt:
+                    return await self._async_extract_output(content_to_extract, llm = llm), current_prompt
+                return await self._async_extract_output(content_to_extract, llm = llm) 
+            time_out += 1
+            
+            # Handle both string prompts and chat message lists
+            llm_response = await llm.async_generate(messages=conversation)
+            conversation.append({"role": "assistant", "content": llm_response.content})
+            
+            # Store the final LLM response
+            final_llm_response = llm_response
+            
+            tool_call_args = self._extract_tool_calls(llm_response.content)
+            if not tool_call_args:
+                break
+            
+            logger.info("Extracted tool call args:")
+            logger.info(json.dumps(tool_call_args, indent=4))
+            
+            results = self._calling_tools(tool_call_args)
+            
+            logger.info("Tool call results:")
+            try:
+                logger.info(json.dumps(results, indent=4))
+            except Exception:
+                logger.info(str(results))
+            
+            conversation.append({"role": "assistant", "content": TOOL_CALLING_HISTORY_PROMPT.format(
+                iteration_number=time_out,
+                tool_call_args=f"{tool_call_args}",
+                results=f"{results}"
+            )})
+        
+        # Get the appropriate prompt for return
+        current_prompt = self.prepare_action_prompt(inputs=prompt_params_values or {})
+        # Use the final LLM response if available, otherwise fall back to execution history
+        content_to_extract = final_llm_response if final_llm_response is not None else "{content}".format(content = conversation)
+        if return_prompt:
+            return await self._async_extract_output(content_to_extract, llm = llm), current_prompt
+        return await self._async_extract_output(content_to_extract, llm = llm)

evoagentx/actions/task_planning.py

@@ -0,0 +1,80 @@
+from pydantic import Field
+from typing import Optional, List
+
+from ..core.logging import logger
+from ..models.base_model import BaseLLM
+from .action import Action, ActionInput, ActionOutput
+from ..prompts.task_planner import TASK_PLANNING_ACTION
+from ..workflow.workflow_graph import WorkFlowNode
+
+
+class TaskPlanningInput(ActionInput):
+    """
+    Input specification for the task planning action.
+    """
+    goal: str = Field(description="A clear and detailed description of the user's goal, specifying what needs to be achieved.")
+    history: Optional[str] = Field(default=None, description="Optional field containing previously generated task plan.")
+    suggestion: Optional[str] = Field(default=None, description="Optional suggestions or ideas to guide the planning process.")
+
+
+class TaskPlanningOutput(ActionOutput):
+    """
+    Output structure for the task planning action.
+    """
+    sub_tasks: List[WorkFlowNode] = Field(description="A list of sub-tasks that collectively achieve user's goal.")
+    
+
+class TaskPlanning(Action):
+    """
+    Action for planning a series of tasks to achieve a goal.
+    """
+
+    def __init__(self, **kwargs):
+
+        name = kwargs.pop("name") if "name" in kwargs else TASK_PLANNING_ACTION["name"]
+        description = kwargs.pop("description") if "description" in kwargs else TASK_PLANNING_ACTION["description"]
+        prompt = kwargs.pop("prompt") if "prompt" in kwargs else TASK_PLANNING_ACTION["prompt"]
+        # inputs_format = kwargs.pop("inputs_format") if "inputs_format" in kwargs else TaskPlanningInput
+        # outputs_format = kwargs.pop("outputs_format") if "outputs_format" in kwargs else TaskPlanningOutput
+        inputs_format = kwargs.pop("inputs_format", None) or TaskPlanningInput
+        outputs_format = kwargs.pop("outputs_format", None) or TaskPlanningOutput
+        super().__init__(name=name, description=description, prompt=prompt, inputs_format=inputs_format, outputs_format=outputs_format, **kwargs)
+    
+    def execute(self, llm: Optional[BaseLLM] = None, inputs: Optional[dict] = None, sys_msg: Optional[str]=None, return_prompt: bool = False, **kwargs) -> TaskPlanningOutput:
+        """Execute the task planning process.
+        
+        This method uses the provided language model to generate a structured
+        plan of sub-tasks based on the user's goal and any additional context.
+        
+        Args:
+            llm: The language model to use for planning.
+            inputs: Input data containing the goal and optional context.
+            sys_msg: Optional system message for the language model.
+            return_prompt: Whether to return both the task plan and the prompt used.
+            **kwargs: Additional keyword arguments.
+            
+        Returns:
+            If return_prompt is False (default): The generated task plan.
+            If return_prompt is True: A tuple of (task plan, prompt used).
+            
+        Raises:
+            ValueError: If the inputs are None or empty.
+        """
+        if not inputs:
+            logger.error("TaskPlanning action received invalid `inputs`: None or empty.")
+            raise ValueError('The `inputs` to TaskPlanning action is None or empty.')
+
+        prompt_params_names = ["goal", "history", "suggestion"]
+        prompt_params_values = {param: inputs.get(param, "") for param in prompt_params_names}
+        prompt = self.prompt.format(**prompt_params_values)
+        task_plan = llm.generate(
+            prompt = prompt, 
+            system_message = sys_msg, 
+            parser=self.outputs_format,
+            parse_mode="json"
+        )
+        
+        if return_prompt:
+            return task_plan, prompt
+        
+        return task_plan

evoagentx/agents/__init__.py

@@ -0,0 +1,6 @@
+from .agent import Agent
+from .customize_agent import CustomizeAgent
+from .action_agent import ActionAgent
+from .agent_manager import AgentManager
+
+__all__ = ["Agent", "CustomizeAgent", "ActionAgent", "AgentManager"]

evoagentx/agents/action_agent.py

@@ -0,0 +1,502 @@
+import asyncio
+import json
+from pydantic import create_model, Field
+from typing import Optional, Callable, Type, List, Any
+
+from .agent import Agent
+from ..core.logging import logger
+from ..core.registry import MODULE_REGISTRY, ACTION_FUNCTION_REGISTRY
+from ..models.model_configs import LLMConfig 
+from ..actions.action import Action, ActionOutput, ActionInput
+from ..utils.utils import generate_dynamic_class_name, make_parent_folder
+from ..core.message import Message, MessageType
+
+
+class ActionAgent(Agent):
+    """
+    ActionAgent is a specialized agent that executes a provided function directly without LLM.
+    It creates an action that uses the provided function as the execution backbone.
+    
+    Attributes:
+        name (str): The name of the agent.
+        description (str): A description of the agent's purpose and capabilities.
+        inputs (List[dict]): List of input specifications, where each dict contains:
+            - name (str): Name of the input parameter
+            - type (str): Type of the input
+            - description (str): Description of what the input represents
+            - required (bool, optional): Whether this input is required (default: True)
+        outputs (List[dict]): List of output specifications, where each dict contains:
+            - name (str): Name of the output field
+            - type (str): Type of the output
+            - description (str): Description of what the output represents
+            - required (bool, optional): Whether this output is required (default: True)
+        execute_func (Callable): The function to execute the agent.
+        async_execute_func (Callable, Optional): Async version of the function. If not provided,
+            an async wrapper will be automatically created around execute_func.
+        llm_config (LLMConfig, optional): Configuration for the language model (minimal usage).
+    """
+    
+    
+    def __init__(
+        self,
+        name: str,
+        description: str,
+        inputs: List[dict],
+        outputs: List[dict],
+        execute_func: Callable,
+        async_execute_func: Optional[Callable] = None,
+        llm_config: Optional[LLMConfig] = None,
+        **kwargs
+    ):
+        # Validate inputs
+        if not callable(execute_func):
+            raise ValueError("execute_func must be callable")
+        
+        if async_execute_func is not None and not callable(async_execute_func):
+            raise ValueError("async_execute_func must be callable")
+        
+        # Validate inputs and outputs
+        self._validate_inputs_outputs(inputs, outputs)
+        
+        # Set is_human based on LLM availability
+        is_human = llm_config is None
+        
+        # Initialize parent directly
+        super().__init__(
+            name=name,
+            description=description,
+            llm_config=llm_config,
+            is_human=is_human,
+            **kwargs
+        )
+        
+        # Store function references and metadata
+        self.execute_func = execute_func
+        self.async_execute_func = async_execute_func
+        self.inputs = inputs
+        self.outputs = outputs
+        
+        # Create and add the function-based action
+        action = self._create_function_action_with_params(
+            name, execute_func, async_execute_func, inputs, outputs
+        )
+        self.add_action(action)
+    
+    def init_llm(self):
+        pass
+    
+    def _validate_inputs_outputs(self, inputs: List[dict], outputs: List[dict]):
+        """Validate the structure of inputs and outputs."""
+        # Allow empty inputs for functions that don't require any inputs
+        if inputs is None:
+            inputs = []
+        
+        if outputs is None:
+            outputs = []
+        
+        # Validate inputs structure
+        for i, input_field in enumerate(inputs):
+            if not isinstance(input_field, dict):
+                raise ValueError(f"Input field {i} must be a dictionary, got {type(input_field)}")
+            
+            required_keys = ["name", "type", "description"]
+            for key in required_keys:
+                if key not in input_field:
+                    raise ValueError(f"Input field {i} missing required key '{key}'")
+            
+            if not isinstance(input_field["name"], str):
+                raise ValueError(f"Input field {i} 'name' must be a string, got {type(input_field['name'])}")
+            
+            if not isinstance(input_field["type"], str):
+                raise ValueError(f"Input field {i} 'type' must be a string, got {type(input_field['type'])}")
+            
+            if not isinstance(input_field["description"], str):
+                raise ValueError(f"Input field {i} 'description' must be a string, got {type(input_field['description'])}")
+            
+            # Check for duplicate input names
+            input_names = [field["name"] for field in inputs]
+            if len(input_names) != len(set(input_names)):
+                raise ValueError(f"Duplicate input names found: {[name for name in input_names if input_names.count(name) > 1]}")
+        
+        # Validate outputs structure
+        for i, output_field in enumerate(outputs):
+            if not isinstance(output_field, dict):
+                raise ValueError(f"Output field {i} must be a dictionary, got {type(output_field)}")
+            
+            required_keys = ["name", "type", "description"]
+            for key in required_keys:
+                if key not in output_field:
+                    raise ValueError(f"Output field {i} missing required key '{key}'")
+            
+            if not isinstance(output_field["name"], str):
+                raise ValueError(f"Output field {i} 'name' must be a string, got {type(output_field['name'])}")
+            
+            if not isinstance(output_field["type"], str):
+                raise ValueError(f"Output field {i} 'type' must be a string, got {type(output_field['type'])}")
+            
+            if not isinstance(output_field["description"], str):
+                raise ValueError(f"Output field {i} 'description' must be a string, got {type(output_field['description'])}")
+            
+            # Check for duplicate output names
+            output_names = [field["name"] for field in outputs]
+            if len(output_names) != len(set(output_names)):
+                raise ValueError(f"Duplicate output names found: {[name for name in output_names if output_names.count(name) > 1]}")
+    
+    def _create_function_action_input_type(self, name: str, inputs: List[dict]) -> Type[ActionInput]:
+        """Create ActionInput type from input specifications."""
+        action_input_fields = {}
+        for field in inputs:
+            required = field.get("required", True)
+            if required:
+                action_input_fields[field["name"]] = (str, Field(description=field["description"]))
+            else:
+                action_input_fields[field["name"]] = (Optional[str], Field(default=None, description=field["description"]))
+        
+        action_input_type = create_model(
+            self._get_unique_class_name(
+                generate_dynamic_class_name(f"{name} action_input")
+            ),
+            **action_input_fields,
+            __base__=ActionInput
+        )
+        return action_input_type
+    
+    def _create_function_action_output_type(self, name: str, outputs: List[dict]) -> Type[ActionOutput]:
+        """Create ActionOutput type from output specifications."""
+        action_output_fields = {}
+        for field in outputs:
+            required = field.get("required", True)
+            if required:
+                action_output_fields[field["name"]] = (Any, Field(description=field["description"]))
+            else:
+                action_output_fields[field["name"]] = (Optional[Any], Field(default=None, description=field["description"]))
+        
+        action_output_type = create_model(
+            self._get_unique_class_name(
+                generate_dynamic_class_name(f"{name} action_output")
+            ),
+            **action_output_fields,
+            __base__=ActionOutput
+        )
+        return action_output_type
+    
+    def _create_execute_method(self, execute_func: Callable):
+        """Create the execute method for the action."""
+        def execute_method(action_self, llm=None, inputs=None, sys_msg=None, return_prompt=False, **kwargs):
+            # Validate inputs
+            if inputs is None:
+                inputs = {}
+            
+            # Validate that all required inputs are provided
+            required_inputs = action_self.inputs_format.get_required_input_names()
+            missing_inputs = [input_name for input_name in required_inputs if input_name not in inputs]
+            if missing_inputs:
+                raise ValueError(f"Missing required inputs: {missing_inputs}")
+            
+            # Validate input types (basic validation)
+            filtered_inputs = {}
+            for input_name, input_value in inputs.items():
+                if input_name in [field["name"] for field in self.inputs]:
+                    filtered_inputs[input_name] = input_value
+                else:
+                    logger.warning(f"Unexpected input '{input_name}' provided")
+            
+            # Execute function
+            try:
+                result = execute_func(**filtered_inputs)
+            except Exception as e:
+                # Create error output - try to use error field if it exists, otherwise use first available field
+                try:
+                    # Check if output format has an error field
+                    output_fields = action_self.outputs_format.get_attrs()
+                    if "error" in output_fields:
+                        error_output = action_self.outputs_format(
+                            error=f"Function execution failed: {str(e)}"
+                        )
+                    elif len(output_fields) > 0:
+                        # Use the first field as error field
+                        first_field = output_fields[0]
+                        error_output = action_self.outputs_format(**{first_field: f"Error: {str(e)}"})
+                    else:
+                        # Fallback to creating a simple output with error message
+                        error_output = action_self.outputs_format()
+                except Exception as create_error:
+                    # If all else fails, create a minimal output
+                    logger.error(f"Failed to create error output: {create_error}")
+                    error_output = action_self.outputs_format()
+                return error_output, "Function execution"
+            
+            # Create success output using the parse method
+            if isinstance(result, dict):
+                # For dict results, create output directly
+                output = action_self.outputs_format(**result)
+            else:
+                # For simple values, create output with the first field
+                output_fields = action_self.outputs_format.get_attrs()
+                if len(output_fields) > 0:
+                    first_field = output_fields[0]
+                    output = action_self.outputs_format(**{first_field: result})
+                else:
+                    # Fallback to creating empty output
+                    output = action_self.outputs_format()
+            
+            return output, "Function execution"
+        
+        return execute_method
+    
+    def _create_async_execute_method(self, async_execute_func: Callable, execute_func: Callable):
+        """Create the async execute method for the action."""
+        async def async_execute_method(action_self, llm=None, inputs=None, sys_msg=None, return_prompt=False, **kwargs):
+            # Validate inputs
+            if inputs is None:
+                inputs = {}
+            
+            # Validate that all required inputs are provided
+            required_inputs = action_self.inputs_format.get_required_input_names()
+            missing_inputs = [input_name for input_name in required_inputs if input_name not in inputs]
+            if missing_inputs:
+                raise ValueError(f"Missing required inputs: {missing_inputs}")
+            
+            # Validate input types (basic validation)
+            filtered_inputs = {}
+            for input_name, input_value in inputs.items():
+                if input_name in [field["name"] for field in self.inputs]:
+                    filtered_inputs[input_name] = input_value
+                else:
+                    logger.warning(f"Unexpected input '{input_name}' provided")
+            
+            # Execute async function
+            try:
+                if async_execute_func is not None:
+                    result = await async_execute_func(**filtered_inputs)
+                else:
+                    # Use sync function in async context
+                    loop = asyncio.get_event_loop()
+                    result = await loop.run_in_executor(None, lambda: execute_func(**filtered_inputs))
+            except Exception as e:
+                # Create error output - try to use error field if it exists, otherwise use first available field
+                try:
+                    # Check if output format has an error field
+                    output_fields = action_self.outputs_format.get_attrs()
+                    if "error" in output_fields:
+                        error_output = action_self.outputs_format(
+                            error=f"Async function execution failed: {str(e)}"
+                        )
+                    elif len(output_fields) > 0:
+                        # Use the first field as error field
+                        first_field = list(output_fields.keys())[0]
+                        error_output = action_self.outputs_format(**{first_field: f"Error: {str(e)}"})
+                    else:
+                        # Fallback to creating a simple output with error message
+                        error_output = action_self.outputs_format()
+                except Exception as create_error:
+                    # If all else fails, create a minimal output
+                    logger.error(f"Failed to create error output: {create_error}")
+                    error_output = action_self.outputs_format()
+                return error_output, "Async function execution"
+            
+            # Create success output using the parse method
+            if isinstance(result, dict):
+                # For dict results, create output directly
+                output = action_self.outputs_format(**result)
+            else:
+                # For simple values, create output with the first field
+                output_fields = action_self.outputs_format.get_attrs()
+                if len(output_fields) > 0:
+                    first_field = output_fields[0]
+                    output = action_self.outputs_format(**{first_field: result})
+                else:
+                    # Fallback to creating empty output
+                    output = action_self.outputs_format()
+            
+            return output, "Async function execution"
+        
+        return async_execute_method
+    
+    def _create_function_action_with_params(self, name: str, execute_func: Callable, async_execute_func: Callable, inputs: List[dict], outputs: List[dict]) -> Action:
+        """Create an action that executes the provided function with given parameters."""
+        
+        # Create input/output types
+        action_input_type = self._create_function_action_input_type(name, inputs)
+        action_output_type = self._create_function_action_output_type(name, outputs)
+        
+        # Create custom action class
+        action_cls_name = self._get_unique_class_name(
+            generate_dynamic_class_name(f"{name} function action")
+        )
+        
+        # Create action class with function execution
+        function_action_cls = create_model(
+            action_cls_name,
+            __base__=Action
+        )
+        
+        # Create action instance
+        function_action = function_action_cls(
+            name=action_cls_name,
+            description=f"Executes {execute_func.__name__} function",
+            inputs_format=action_input_type,
+            outputs_format=action_output_type
+        )
+        
+        # Override execute methods - bind them properly to the action instance
+        execute_method = self._create_execute_method(execute_func)
+        async_execute_method = self._create_async_execute_method(async_execute_func, execute_func)
+        
+        # Bind the methods to the action instance
+        function_action.execute = execute_method.__get__(function_action, type(function_action))
+        function_action.async_execute = async_execute_method.__get__(function_action, type(function_action))
+        
+        return function_action
+    
+    def _create_function_action(self, name: str, execute_func: Callable, async_execute_func: Callable, inputs: List[dict], outputs: List[dict]) -> Action:
+        """Create an action that executes the provided function."""
+        return self._create_function_action_with_params(
+            name,
+            execute_func, 
+            async_execute_func, 
+            inputs, 
+            outputs
+        )
+    
+    def get_config(self) -> dict:
+        """Get configuration for the ActionAgent."""
+        # Get base config from Agent
+        config = super().get_config()
+        
+        # Add ActionAgent-specific information
+        config.update({
+            "class_name": "ActionAgent",
+            "execute_func_name": self.execute_func.__name__ if self.execute_func else None,
+            "async_execute_func_name": self.async_execute_func.__name__ if self.async_execute_func else None,
+            "inputs": self.inputs,
+            "outputs": self.outputs
+        })
+        return config
+    
+    def save_module(self, path: str, ignore: List[str] = [], **kwargs) -> str:
+        """Save the ActionAgent configuration to a JSON file.
+        
+        Args:
+            path: File path where the configuration should be saved
+            ignore: List of keys to exclude from the saved configuration
+            **kwargs (Any): Additional parameters for the save operation
+            
+        Returns:
+            The path where the configuration was saved
+        """
+        config = self.get_config()
+        
+        # Add ActionAgent-specific information
+        config.update({
+            "class_name": "ActionAgent",
+            "execute_func_name": self.execute_func.__name__ if self.execute_func else None,
+            "async_execute_func_name": self.async_execute_func.__name__ if self.async_execute_func else None,
+            "inputs": self.inputs,
+            "outputs": self.outputs
+        })
+        
+        # Remove non-serializable items
+        for ignore_key in ignore:
+            config.pop(ignore_key, None)
+        
+        # Save to JSON file
+        make_parent_folder(path)
+        with open(path, 'w', encoding='utf-8') as f:
+            json.dump(config, f, indent=4, ensure_ascii=False)
+        
+        return path
+    
+    @classmethod
+    def load_module(cls, path: str, llm_config: LLMConfig = None, **kwargs) -> "ActionAgent":
+        """Load the ActionAgent from a JSON file.
+        
+        Args:
+            path: The path of the file
+            llm_config: The LLMConfig instance (optional)
+            **kwargs: Additional keyword arguments
+            
+        Returns:
+            ActionAgent: The loaded agent instance
+            
+        Raises:
+            KeyError: If required functions are not found in the registry
+        """
+        # Load configuration
+        with open(path, 'r', encoding='utf-8') as f:
+            config = json.load(f)
+        
+        # Extract function names
+        execute_func_name = config.get("execute_func_name")
+        async_execute_func_name = config.get("async_execute_func_name")
+        
+        # Retrieve functions from registry
+        execute_func = None
+        async_execute_func = None
+        
+        if execute_func_name:
+            if not ACTION_FUNCTION_REGISTRY.has_function(execute_func_name):
+                raise KeyError(f"Function '{execute_func_name}' not found in registry. Please register it first.")
+            execute_func = ACTION_FUNCTION_REGISTRY.get_function(execute_func_name)
+        
+        if async_execute_func_name:
+            if not ACTION_FUNCTION_REGISTRY.has_function(async_execute_func_name):
+                raise KeyError(f"Function '{async_execute_func_name}' not found in registry. Please register it first.")
+            async_execute_func = ACTION_FUNCTION_REGISTRY.get_function(async_execute_func_name)
+        
+        # Create agent
+        agent = cls(
+            name=config["name"],
+            description=config["description"],
+            inputs=config["inputs"],
+            outputs=config["outputs"],
+            execute_func=execute_func,
+            async_execute_func=async_execute_func,
+            llm_config=llm_config,
+            **kwargs
+        )
+        
+        return agent
+    
+    def __call__(self, inputs: dict = None, return_msg_type: MessageType = MessageType.UNKNOWN, **kwargs) -> Message:
+        """
+        Call the main function action.
+
+        Args:
+            inputs (dict): The inputs to the function action.
+            return_msg_type (MessageType): The type of message to return.
+            **kwargs (Any): Additional keyword arguments.
+
+        Returns:
+            Message: The output of the function action.
+        """
+        inputs = inputs or {} 
+        return super().__call__(action_name=self.main_action_name, action_input_data=inputs, return_msg_type=return_msg_type, **kwargs)
+    
+    @property
+    def main_action_name(self) -> str:
+        """
+        Get the name of the main function action for this agent.
+        
+        Returns:
+            The name of the main function action
+        """
+        for action in self.actions:
+            if action.name != self.cext_action_name:
+                return action.name
+        raise ValueError("Couldn't find the main action name!")
+    
+    def _get_unique_class_name(self, candidate_name: str) -> str:
+        """
+        Get a unique class name by checking if it already exists in the registry.
+        If it does, append "Vx" to make it unique.
+        """
+        if not MODULE_REGISTRY.has_module(candidate_name):
+            return candidate_name
+        
+        counter = 1
+        while True:
+            new_name = f"{candidate_name}V{counter}"
+            if not MODULE_REGISTRY.has_module(new_name):
+                return new_name
+            counter += 1

evoagentx/agents/agent.py

@@ -0,0 +1,531 @@
+import asyncio
+import inspect 
+from pydantic import Field
+from typing import Type, Optional, Union, Tuple, List, Any, Coroutine
+
+from ..core.module import BaseModule
+from ..core.module_utils import generate_id
+from ..core.message import Message, MessageType
+from ..core.registry import MODEL_REGISTRY
+from ..models.model_configs import LLMConfig
+from ..models.base_model import BaseLLM
+from ..memory.memory import ShortTermMemory
+from ..memory.long_term_memory import LongTermMemory
+from ..memory.memory_manager import MemoryManager
+from ..storages.base import StorageHandler
+from ..actions.action import Action
+from ..actions.action import ContextExtraction
+
+
+class Agent(BaseModule):
+    """
+    Base class for all agents. 
+    
+    Attributes:
+        name (str): Unique identifier for the agent
+        description (str): Human-readable description of the agent's purpose
+        llm_config (Optional[LLMConfig]): Configuration for the language model. If provided, a new LLM instance will be created. 
+            Otherwise, the existing LLM instance specified in the `llm` field will be used.   
+        llm (Optional[BaseLLM]): Language model instance. If provided, the existing LLM instance will be used. 
+        agent_id (Optional[str]): Unique ID for the agent, auto-generated if not provided
+        system_prompt (Optional[str]): System prompt for the Agent.
+        actions (List[Action]): List of available actions
+        n (Optional[int]): Number of latest messages used to provide context for action execution. It uses all the messages in short term memory by default. 
+        is_human (bool): Whether this agent represents a human user
+        version (int): Version number of the agent, default is 0. 
+    """
+
+    name: str # should be unique
+    description: str
+    llm_config: Optional[LLMConfig] = None
+    llm: Optional[BaseLLM] = None
+    agent_id: Optional[str] = Field(default_factory=generate_id)
+    system_prompt: Optional[str] = None
+    short_term_memory: Optional[ShortTermMemory] = Field(default_factory=ShortTermMemory) # store short term memory for a single workflow.
+    use_long_term_memory: Optional[bool] = False
+    storage_handler: Optional[StorageHandler] = None
+    long_term_memory: Optional[LongTermMemory] = None
+    long_term_memory_manager: Optional[MemoryManager] = None
+    actions: List[Action] = Field(default=None)
+    n: int = Field(default=None, description="number of latest messages used to provide context for action execution. It uses all the messages in short term memory by default.")
+    is_human: bool = Field(default=False)
+    version: int = 0 
+
+    def init_module(self):
+        if not self.is_human:
+            self.init_llm()
+        if self.use_long_term_memory:
+            self.init_long_term_memory()
+        self.actions = [] if self.actions is None else self.actions
+        self._action_map = {action.name: action for action in self.actions} if self.actions else dict()
+        self._save_ignore_fields = ["llm", "llm_config"]
+        self.init_context_extractor()
+
+    # def __call__(self, *args, **kwargs) -> Message:
+    #     """Make the agent callable and automatically choose between sync and async execution"""
+    #     if asyncio.iscoroutinefunction(self.async_execute) and asyncio.get_event_loop().is_running():
+    #         # If the operator is in an asynchronous environment and has an execute_async method, return a coroutine
+    #         return self.async_execute(*args, **kwargs)
+    #     # Otherwise, use the synchronous method
+    #     return self.execute(*args, **kwargs)
+
+    def __call__(self, *args: Any, **kwargs: Any) -> Union[dict, Coroutine[Any, Any, dict]]:
+        """Make the operator callable and automatically choose between sync and async execution."""
+        try:
+            # Safe way to check if we're inside an async environment
+            asyncio.get_running_loop()
+            return self.async_execute(*args, **kwargs)
+        except RuntimeError:
+            # No running loop — likely in sync context or worker thread
+            return self.execute(*args, **kwargs)
+    
+    def _prepare_execution(
+        self,
+        action_name: str,
+        msgs: Optional[List[Message]] = None,
+        action_input_data: Optional[dict] = None,
+        **kwargs
+    ) -> Tuple[Action, dict]:
+        """Prepare for action execution by updating memory and getting inputs.
+        
+        Helper method used by both execute and aexecute methods.
+        
+        Args:
+            action_name: The name of the action to execute
+            msgs: Optional list of messages providing context for the action
+            action_input_data: Optional pre-extracted input data for the action
+            **kwargs: Additional workflow parameters
+            
+        Returns:
+            Tuple containing the action object and input data
+            
+        Raises:
+            AssertionError: If neither msgs nor action_input_data is provided
+        """
+        assert msgs is not None or action_input_data is not None, "must provide either `msgs` or `action_input_data`"
+        action = self.get_action(action_name=action_name)
+
+        # update short-term memory
+        if msgs is not None:
+            # directly add messages to short-term memory
+            self.short_term_memory.add_messages(msgs)
+        if action_input_data is not None:
+            # create a message from action_input_data and add it to short-term memory
+            input_message = Message(
+                content = action_input_data,
+                next_actions = [action_name],
+                msg_type = MessageType.INPUT, 
+                wf_goal = kwargs.get("wf_goal", None),
+                wf_task = kwargs.get("wf_task", None),
+                wf_task_desc = kwargs.get("wf_task_desc", None)
+            )
+            self.short_term_memory.add_message(input_message)
+        
+        # obtain action input data from short term memory if not provided
+        action_input_data = action_input_data or self.get_action_inputs(action=action)
+        
+        return action, action_input_data
+    
+    def _create_output_message(
+        self,
+        action_output,
+        prompt: str,
+        action_name: str,
+        return_msg_type: Optional[MessageType] = MessageType.UNKNOWN,
+        **kwargs
+    ) -> Message:
+        """Create a message from execution results and update memory.
+        
+        Helper method used by both execute and aexecute methods.
+        
+        Args:
+            action_output: The output from action execution
+            prompt: The prompt used for execution
+            action_name: The name of the executed action
+            return_msg_type: Message type for the return message
+            **kwargs: Additional workflow parameters
+            
+        Returns:
+            Message object containing execution results
+        """
+        # formulate a message
+        message = Message(
+            content=action_output, 
+            agent=self.name,
+            action=action_name,
+            prompt=prompt, 
+            msg_type=return_msg_type,
+            wf_goal = kwargs.get("wf_goal", None),
+            wf_task = kwargs.get("wf_task", None),
+            wf_task_desc = kwargs.get("wf_task_desc", None)
+        )
+
+        # update short-term memory
+        self.short_term_memory.add_message(message)
+        
+        return message
+    
+    async def async_execute(
+        self, 
+        action_name: str, 
+        msgs: Optional[List[Message]] = None, 
+        action_input_data: Optional[dict] = None, 
+        return_msg_type: Optional[MessageType] = MessageType.UNKNOWN,
+        return_action_input_data: Optional[bool] = False, 
+        **kwargs
+    ) -> Union[Message, Tuple[Message, dict]]:
+        """Execute an action asynchronously with the given context and return results.
+
+        This is the async version of the execute method, allowing it to perform actions
+        based on the current conversation context.
+
+        Args:
+            action_name: The name of the action to execute
+            msgs: Optional list of messages providing context for the action
+            action_input_data: Optional pre-extracted input data for the action
+            return_msg_type: Message type for the return message
+            **kwargs (Any): Additional parameters, may include workflow information
+        
+        Returns:
+            Message: A message containing the execution results
+        """
+        action, action_input_data = self._prepare_execution(
+            action_name=action_name,
+            msgs=msgs,
+            action_input_data=action_input_data,
+            **kwargs
+        )
+
+        # execute action asynchronously
+        async_execute_source = inspect.getsource(action.async_execute)
+        if "NotImplementedError" in async_execute_source:
+            # if the async_execute method is not implemented, use the execute method instead
+            execution_results = action.execute(
+                llm=self.llm, 
+                inputs=action_input_data, 
+                sys_msg=self.system_prompt,
+                return_prompt=True,
+                **kwargs
+            )
+        else:
+            execution_results = await action.async_execute(
+                llm=self.llm, 
+                inputs=action_input_data, 
+                sys_msg=self.system_prompt,
+                return_prompt=True,
+                **kwargs
+        )
+        action_output, prompt = execution_results
+
+        message = self._create_output_message(
+            action_output=action_output,
+            prompt=prompt,
+            action_name=action_name,
+            return_msg_type=return_msg_type,
+            **kwargs
+        )
+        if return_action_input_data:
+            return message, action_input_data
+        return message
+    
+    def execute(
+        self, 
+        action_name: str, 
+        msgs: Optional[List[Message]] = None, 
+        action_input_data: Optional[dict] = None, 
+        return_msg_type: Optional[MessageType] = MessageType.UNKNOWN,
+        return_action_input_data: Optional[bool] = False, 
+        **kwargs
+    ) -> Union[Message, Tuple[Message, dict]]:
+        """Execute an action with the given context and return results.
+
+        This is the core method for agent functionality, allowing it to perform actions
+        based on the current conversation context.
+
+        Args:
+            action_name: The name of the action to execute
+            msgs: Optional list of messages providing context for the action
+            action_input_data: Optional pre-extracted input data for the action
+            return_msg_type: Message type for the return message
+            **kwargs (Any): Additional parameters, may include workflow information
+        
+        Returns:
+            Message: A message containing the execution results
+        """
+        action, action_input_data = self._prepare_execution(
+            action_name=action_name,
+            msgs=msgs,
+            action_input_data=action_input_data,
+            **kwargs
+        )
+
+        # execute action
+        execution_results = action.execute(
+            llm=self.llm, 
+            inputs=action_input_data, 
+            sys_msg=self.system_prompt,
+            return_prompt=True,
+            **kwargs
+        )
+        action_output, prompt = execution_results
+
+        message = self._create_output_message(
+            action_output=action_output,
+            prompt=prompt,
+            action_name=action_name,
+            return_msg_type=return_msg_type,
+            **kwargs
+        )
+        if return_action_input_data:
+            return message, action_input_data
+        return message
+    
+    def init_llm(self):
+        """
+        Initialize the language model for the agent.
+        """
+        # Only initialize LLM if not human and LLM is provided
+        if not self.is_human and (not self.llm_config and not self.llm):
+            raise ValueError("must provide `llm_config` or `llm` when `is_human` is False")
+        if not self.is_human and (self.llm_config or self.llm):
+            if self.llm_config and not self.llm:
+                llm_cls = MODEL_REGISTRY.get_model(self.llm_config.llm_type)
+                self.llm = llm_cls(config=self.llm_config)
+            if self.llm:
+                self.llm_config = self.llm.config
+        # If is_human=True or no LLM provided, self.llm remains None
+
+    def init_long_term_memory(self):
+        """
+        Initialize long-term memory components.
+        """
+        assert self.storage_handler is not None, "must provide ``storage_handler`` when use_long_term_memory=True"
+        # TODO revise the initialisation of long_term_memory and long_term_memory_manager
+        if not self.long_term_memory:
+            self.long_term_memory = LongTermMemory()
+        if not self.long_term_memory_manager:
+            self.long_term_memory_manager = MemoryManager(
+                storage_handler=self.storage_handler,
+                memory=self.long_term_memory
+            )
+    
+    def init_context_extractor(self):
+        """
+        Initialize the context extraction action.
+        """
+        cext_action = ContextExtraction()
+        self.cext_action_name = cext_action.name
+        self.add_action(cext_action)
+
+    def add_action(self, action: Type[Action]):
+        """
+        Add a new action to the agent's available actions.
+
+        Args:
+            action: The action instance to add
+        """
+        action_name  = action.name
+        if action_name in self._action_map:
+            return
+        self.actions.append(action)
+        self._action_map[action_name] = action
+
+    def check_action_name(self, action_name: str):
+        """
+        Check if an action name is valid for this agent.
+                
+        Args:
+            action_name: Name of the action to check
+        """
+        if action_name not in self._action_map:
+            raise KeyError(f"'{action_name}' is an invalid action for {self.name}! Available action names: {list(self._action_map.keys())}")
+    
+    def get_action(self, action_name: str) -> Action:
+        """
+        Retrieves the Action instance associated with the given name.
+        
+        Args:
+            action_name: Name of the action to retrieve
+            
+        Returns:
+            The Action instance with the specified name
+        """
+        self.check_action_name(action_name=action_name)
+        return self._action_map[action_name]
+    
+    def get_action_name(self, action_cls: Type[Action]) -> str:
+        """
+        Searches through the agent's actions to find one matching the specified type.
+        
+        Args:
+            action_cls: The Action class type to search for
+            
+        Returns:
+            The name of the matching action
+        """
+        for name, action in self._action_map.items():
+            if isinstance(action, action_cls):
+                return name
+        raise ValueError(f"Couldn't find an action that matches Type '{action_cls.__name__}'")
+    
+    def get_action_inputs(self, action: Action) -> Union[dict, None]:
+        """
+        Uses the context extraction action to determine appropriate inputs
+        for the specified action based on the conversation history.
+        
+        Args:
+            action: The action for which to extract inputs
+            
+        Returns:
+            Dictionary of extracted input data, or None if extraction fails
+        """
+        # return the input data of an action.
+        context = self.short_term_memory.get(n=self.n)
+        cext_action = self.get_action(self.cext_action_name)
+        action_inputs = cext_action.execute(llm=self.llm, action=action, context=context)
+        return action_inputs
+    
+    def get_all_actions(self) -> List[Action]:
+        """Get all actions except the context extraction action.
+        
+        Returns:
+            List of Action instances available for execution
+        """
+        actions = [action for action in self.actions if action.name != self.cext_action_name]
+        return actions
+    
+    def get_agent_profile(self, action_names: List[str] = None) -> str:
+        """Generate a human-readable profile of the agent and its capabilities.
+        
+        Args:
+            action_names: Optional list of action names to include in the profile.
+                          If None, all actions are included.
+            
+        Returns:
+            A formatted string containing the agent profile
+        """
+        all_actions = self.get_all_actions()
+        if action_names is None:
+            # if `action_names` is None, return description of all actions 
+            action_descriptions = "\n".join([f"  - {action.name}: {action.description}" for action in all_actions])
+        else: 
+            # otherwise, only return description of actions that matches `action_names`
+            action_descriptions = "\n".join([f"  - {action.name}: {action.description}" for action in all_actions if action.name in action_names])
+        profile = f"Agent Name: {self.name}\nDescription: {self.description}\nAvailable Actions:\n{action_descriptions}"
+        return profile
+
+    def clear_short_term_memory(self):
+        """
+        Remove all content from the agent's short-term memory.
+        """
+        pass 
+        
+    def __eq__(self, other: "Agent"):
+        return self.agent_id == other.agent_id
+
+    def __hash__(self):
+        return self.agent_id
+    
+    def get_prompts(self) -> dict:
+        """
+        Get all the prompts of the agent.
+        
+        Returns:
+            dict: A dictionary with keys in the format 'agent_name::action_name' and values
+                containing the system_prompt and action prompt.
+        """
+        prompts = {}
+        for action in self.get_all_actions():
+            prompts[action.name] = {
+                "system_prompt": self.system_prompt, 
+                "prompt": action.prompt
+            }
+        return prompts
+    
+    def set_prompt(self, action_name: str, prompt: str, system_prompt: Optional[str] = None) -> bool:
+        """
+        Set the prompt for a specific action of this agent.
+        
+        Args:
+            action_name: Name of the action whose prompt should be updated
+            prompt: New prompt text to set for the action
+            system_prompt: Optional new system prompt to set for the agent
+            
+        Returns:
+            bool: True if the prompt was successfully updated, False otherwise
+            
+        Raises:
+            KeyError: If the action_name does not exist for this agent
+        """
+        try:
+            action = self.get_action(action_name)
+            action.prompt = prompt
+            
+            if system_prompt is not None:
+                self.system_prompt = system_prompt
+                
+            return True
+        except KeyError:
+            raise KeyError(f"Action '{action_name}' not found in agent '{self.name}'")
+        
+    def set_prompts(self, prompts: dict) -> bool:
+        """
+        Set the prompts for all actions of this agent.
+        
+        Args:
+            prompts: A dictionary with keys in the format 'action_name' and values
+                containing the system_prompt and action prompt.
+        
+        Returns:
+            bool: True if the prompts were successfully updated, False otherwise
+        """
+        for action_name, prompt_data in prompts.items():
+            # self.set_prompt(action_name, prompt_data["prompt"], prompt_data["system_prompt"])
+            if not isinstance(prompt_data, dict):
+                raise ValueError(f"Invalid prompt data for action '{action_name}'. Expected a dictionary with 'prompt' and 'system_prompt' (optional) keys.")
+            if "prompt" not in prompt_data:
+                raise ValueError(f"Missing 'prompt' key in prompt data for action '{action_name}'.")
+            self.set_prompt(action_name, prompt_data["prompt"], prompt_data.get("system_prompt", None))
+        return True
+
+    def save_module(self, path: str, ignore: List[str] = [], **kwargs)-> str:
+        """Save the agent to persistent storage.
+                
+        Args:
+            path: Path where the agent should be saved
+            ignore: List of field names to exclude from serialization
+            **kwargs (Any): Additional parameters for the save operation
+            
+        Returns:
+            The path where the agent was saved
+        """
+        ignore_fields = self._save_ignore_fields + ignore
+        super().save_module(path=path, ignore=ignore_fields, **kwargs)
+
+    @classmethod
+    def load_module(cls, path: str, llm_config: LLMConfig = None, **kwargs) -> "Agent":
+        """
+        load the agent from local storage. Must provide `llm_config` when loading the agent from local storage. 
+
+        Args:
+            path: The path of the file
+            llm_config: The LLMConfig instance
+        
+        Returns:
+            Agent: The loaded agent instance
+        """
+        agent = super().load_module(path=path, **kwargs)
+        if llm_config is not None:
+            agent["llm_config"] = llm_config.to_dict()
+        return agent 
+    
+    def get_config(self) -> dict:
+        """
+        Get a dictionary containing all necessary configuration to recreate this agent.
+        
+        Returns:
+            dict: A configuration dictionary that can be used to initialize a new Agent instance
+            with the same properties as this one.
+        """
+        config = self.to_dict()
+        return config

evoagentx/agents/agent_generator.py

@@ -0,0 +1,23 @@
+from .agent import Agent
+from ..actions.agent_generation import AgentGeneration 
+from ..prompts.agent_generator import AGENT_GENERATOR
+
+
+class AgentGenerator(Agent):
+
+    """
+    An agent responsible for generating agents for a task. 
+    """
+    
+    def __init__(self, **kwargs):
+
+        name = kwargs.pop("name") if "name" in kwargs else AGENT_GENERATOR["name"]
+        description = kwargs.pop("description") if "description" in kwargs else AGENT_GENERATOR["description"]
+        system_prompt = kwargs.pop("system_prompt") if "system_prompt" in kwargs else AGENT_GENERATOR["system_prompt"]
+        actions = kwargs.pop("actions") if "actions" in kwargs else [AgentGeneration(tools=kwargs.pop("tools", []))]
+        super().__init__(name=name, description=description, system_prompt=system_prompt, actions=actions, **kwargs)
+    
+    @property
+    def agent_generation_action_name(self):
+        return self.get_action_name(action_cls=AgentGeneration)
+    

evoagentx/agents/agent_manager.py

@@ -0,0 +1,505 @@
+import threading
+from enum import Enum
+from typing import Union, Optional, Dict, List
+from pydantic import Field
+from copy import deepcopy
+
+from .agent import Agent
+# from .agent_generator import AgentGenerator
+from .customize_agent import CustomizeAgent
+from ..core.module import BaseModule
+from ..core.decorators import atomic_method
+from ..storages.base import StorageHandler
+from ..models.model_configs import LLMConfig
+from ..tools.tool import Toolkit, Tool
+class AgentState(str, Enum):
+    AVAILABLE = "available"
+    RUNNING = "running"
+
+
+class AgentManager(BaseModule):
+    """
+    Responsible for creating and managing all Agent objects required for workflow operation.
+
+    Attributes:
+        storage_handler (StorageHandler): Used to load and save agents from/to storage.
+        agents (List[Agent]): A list to keep track of all managed Agent instances.
+        agent_states (Dict[str, AgentState]): A dictionary to track the state of each Agent by name.
+    """
+    agents: List[Agent] = Field(default_factory=list)
+    agent_states: Dict[str, AgentState] = Field(default_factory=dict) # agent_name to AgentState mapping
+    storage_handler: Optional[StorageHandler] = None # used to load and save agent from storage.
+    # agent_generator: Optional[AgentGenerator] = None # used to generate agents for a specific subtask
+    tools: Optional[List[Union[Toolkit, Tool]]] = None
+
+    def init_module(self):
+        self._lock = threading.Lock()
+        self._state_conditions = {}
+        if self.agents:
+            for agent in self.agents:
+                self.agent_states[agent.name] = self.agent_states.get(agent.name, AgentState.AVAILABLE)
+                if agent.name not in self._state_conditions:
+                    self._state_conditions[agent.name] = threading.Condition()
+            self.check_agents()
+    
+    def check_agents(self):
+        """Validate agent list integrity and state consistency.
+        
+        Performs thorough validation of the agent manager's internal state:
+        1. Checks for duplicate agent names
+        2. Verifies that agent states exist for all agents
+        3. Ensures agent list and state dictionary sizes match
+        """
+        # check that the names of self.agents should be unique
+        duplicate_agent_names = self.find_duplicate_agents(self.agents)
+        if duplicate_agent_names:
+            raise ValueError(f"The agents should be unique. Found duplicate agent names: {duplicate_agent_names}!")
+        # check agent states
+        if len(self.agents) != len(self.agent_states):
+            raise ValueError(f"The lengths of self.agents ({len(self.agents)}) and self.agent_states ({len(self.agent_states)}) are different!")
+        missing_agents = self.find_missing_agent_states()
+        if missing_agents:
+            raise ValueError(f"The following agents' states were not found: {missing_agents}")
+
+    def find_duplicate_agents(self, agents: List[Agent]) -> List[str]:
+        # return the names of duplicate agents based on agent.name 
+        unique_agent_names = set()
+        duplicate_agent_names = set()
+        for agent in agents:
+            agent_name = agent.name
+            if agent_name in unique_agent_names:
+                duplicate_agent_names.add(agent_name)
+            unique_agent_names.add(agent_name)
+        return list(duplicate_agent_names)
+
+    def find_missing_agent_states(self):
+        missing_agents = [agent.name for agent in self.agents if agent.name not in self.agent_states]
+        return missing_agents
+
+    def list_agents(self) -> List[str]:
+        return [agent.name for agent in self.agents]
+    
+    def has_agent(self, agent_name: str) -> bool:
+        """Check if an agent with the given name exists in the manager.
+        
+        Args:
+            agent_name: The name of the agent to check
+            
+        Returns:
+            True if an agent with the given name exists, False otherwise
+        """
+        all_agent_names = self.list_agents()
+        return agent_name in all_agent_names
+    
+    @property
+    def size(self):
+        """
+        Get the total number of agents managed by this manager.
+        """
+        return len(self.agents)
+    
+    def load_agent(self, agent_name: str, **kwargs) -> Agent:
+        """Load an agent from local storage through storage_handler.
+        
+        Retrieves agent data from storage and creates an Agent instance.
+        
+        Args:
+            agent_name: The name of the agent to load
+            **kwargs (Any): Additional parameters for agent creation
+        
+        Returns:
+            Agent instance with data loaded from storage
+        """
+        if not self.storage_handler:
+            raise ValueError("must provide ``self.storage_handler`` to use ``load_agent``")
+        agent_data = self.storage_handler.load_agent(agent_name=agent_name)
+        agent: Agent = self.create_customize_agent(agent_data=agent_data)
+        return agent
+
+    def load_all_agents(self, **kwargs):
+        """Load all agents from storage and add them to the manager.
+        
+        Retrieves all available agents from storage and adds them to the
+        managed agents collection.
+        
+        Args:
+            **kwargs (Any): Additional parameters passed to storage handler
+        """
+        pass 
+    
+    def update_tools(self, agent_data: dict) -> None:
+        """
+        Update agent_data with tools based on tool_names.
+        
+        Handles four scenarios:
+        1. Neither tool_names nor tools exist: return directly
+        2. Only tool_names exists: resolve tool_names to tools and set tools field
+        3. Only tools exists: return directly (no action needed)
+        4. Both exist: merge tool_names into existing tools (skip duplicates)
+        
+        Args:
+            agent_data (dict): Agent configuration dictionary that may contain 'tool_names' and/or 'tools'
+            
+        Raises:
+            ValueError: If tool_names exist but self.tools is None, or if requested tools are not found
+        """
+        tool_names = agent_data.get("tool_names", None)
+        existing_tools = agent_data.get("tools", None)
+        
+        # Case 1: Neither tool_names nor tools exist
+        if not tool_names and not existing_tools:
+            return
+        
+        # Case 3: Only tools exist (no tool_names)
+        if not tool_names and existing_tools:
+            return
+        
+        # For cases 2 and 4: tool_names exists, need to resolve
+        if self.tools is None:
+            raise ValueError(
+                f"Agent requires tools {tool_names}, but no tools are available in AgentManager. "
+                f"Please set self.tools before creating agents with tool_names."
+            )
+        
+        # Create tool mapping from available tools
+        tool_mapping = {}
+        for tool in self.tools:
+            tool_mapping[tool.name] = tool
+        
+        # Case 2: Only tool_names exists - initialize empty tools list
+        if tool_names and not existing_tools:
+            existing_tools = []
+        
+        # Case 2 & 4: Process tool_names (either with empty or existing tools list)
+        if tool_names:
+            # Create a set of existing tool names for quick lookup
+            existing_tool_names = {tool.name for tool in existing_tools}
+            
+            tools_to_add = []
+            missing_tools = []
+            
+            for tool_name in tool_names:
+                # Skip if tool already exists in tools
+                if tool_name in existing_tool_names:
+                    continue
+                    
+                # Try to resolve new tool
+                if tool_name in tool_mapping:
+                    tools_to_add.append(tool_mapping[tool_name])
+                else:
+                    missing_tools.append(tool_name)
+            
+            if missing_tools:
+                available_tools = list(tool_mapping.keys())
+                raise ValueError(
+                    f"The following tools are not available: {missing_tools}. "
+                    f"Available tools: {available_tools}"
+                )
+            
+            # Merge new tools with existing ones
+            if tools_to_add:
+                agent_data["tools"] = list(existing_tools) + tools_to_add
+
+    def create_customize_agent(self, agent_data: dict, llm_config: Optional[Union[LLMConfig, dict]]=None, **kwargs) -> CustomizeAgent:
+        """
+        create a customized agent from the provided `agent_data`. 
+
+        Args:
+            agent_data: The data used to create an Agent instance, must contain 'name', 'description' and 'prompt' keys.
+            llm_config (Optional[LLMConfig]): The LLM configuration to be used for the agent. 
+                It will be used as the default LLM for agents without a `llm_config` key. 
+                If not provided, the `agent_data` should contain a `llm_config` key. 
+                If provided and `agent_data` contains a `llm_config` key, the `llm_config` in `agent_data` will be used.  
+            **kwargs (Any): Additional parameters for agent creation
+        
+        Returns:
+            Agent: the instantiated agent instance.
+        """
+        
+        agent_data = deepcopy(agent_data)
+        agent_llm_config = agent_data.get("llm_config", llm_config)
+        if not agent_data.get("is_human", False) and not agent_llm_config:
+            raise ValueError("`agent_data` should contain a `llm_config` key or `llm_config` should be provided.")
+
+        if agent_llm_config:
+            if isinstance(agent_llm_config, dict):
+                agent_data["llm_config"] = agent_llm_config
+            elif isinstance(agent_llm_config, LLMConfig):
+                agent_data["llm_config"] = agent_llm_config.to_dict()
+        
+        # tool_mapping = {}
+        # if self.tools is not None:
+        #     for tool in self.tools:
+        #         tool_mapping[tool.name] = tool
+        # if agent_data.get("tool_names", None):
+        #     agent_data["tools"] = [tool_mapping[tool_name] for tool_name in agent_data["tool_names"]]
+        self.update_tools(agent_data=agent_data) # add `tools` field if needed 
+        return CustomizeAgent.from_dict(data=agent_data)
+    
+    def get_agent_name(self, agent: Union[str, dict, Agent]) -> str:
+        """Extract agent name from different agent representations.
+        
+        Handles different ways to specify an agent (string name, dictionary, or
+        Agent instance) and extracts the agent name.
+        
+        Args:
+            agent: Agent specified as a string name, dictionary with 'name' key,
+                  or Agent instance
+                  
+        Returns:
+            The extracted agent name as a string
+        """
+        if isinstance(agent, str):
+            agent_name = agent
+        elif isinstance(agent, dict):
+            agent_name = agent["name"]
+        elif isinstance(agent, Agent):
+            agent_name = agent.name
+        else:
+            raise ValueError(f"{type(agent)} is not a supported type for ``get_agent_name``. Supported types: [str, dict, Agent].")
+        return agent_name
+    
+    def create_agent(self, agent: Union[str, dict, Agent], llm_config: Optional[LLMConfig]=None, **kwargs) -> Agent:
+
+        if isinstance(agent, str):
+            if self.storage_handler is None:
+                # if self.storage_handler is None, the agent (str) must exist in self.agents. Otherwise, a dictionary or an Agent instance should be provided.
+                if not self.has_agent(agent_name=agent):
+                    raise ValueError(f"Agent ``{agent}`` does not exist! You should provide a dictionary or an Agent instance when ``self.storage_handler`` is not provided.")
+                return self.get_agent(agent_name=agent)
+            else:
+                # if self.storage_handler is not None, the agent (str) must exist in the storage and will be loaded from the storage.
+                agent_instance = self.load_agent(agent_name=agent)
+        elif isinstance(agent, dict):
+            if not agent.get("is_human", False) and (llm_config is None and "llm_config" not in agent):
+                raise ValueError("When providing an agent as a dictionary, you must either include 'llm_config' in the dictionary or provide it as a parameter.")
+            agent_instance = self.create_customize_agent(agent_data=agent, llm_config=llm_config, **kwargs)
+        elif isinstance(agent, Agent):
+            agent_instance = agent
+        else:
+            raise ValueError(f"{type(agent)} is not a supported input type of ``create_agent``. Supported types: [str, dict, Agent].")
+        return agent_instance
+    
+    @atomic_method
+    def add_agent(self, agent: Union[str, dict, Agent], llm_config: Optional[LLMConfig]=None, **kwargs):
+        """
+        add a single agent, ignore if the agent already exists (judged by the name of an agent).
+
+        Args:
+            agent: The agent to be added, specified as:
+                - String: Agent name to load from storage
+                - Dictionary: Agent specification to create a CustomizeAgent
+                - Agent: Existing Agent instance to add directly
+            llm_config (Optional[LLMConfig]): The LLM configuration to be used for the agent. Only used when the `agent` is a dictionary, used to create a CustomizeAgent. 
+            **kwargs (Any): Additional parameters for agent creation
+        """
+        # Check for 'tool' key and convert it to 'tools' if needed
+        # if isinstance(agent, dict) and "tool_names" in agent:
+        #     tools_mapping = {}
+        #     if self.tools is not None:
+        #         for tool in self.tools:
+        #             tools_mapping[tool.name] = tool
+        #     agent["tools"] = [tools_mapping[tool_name] for tool_name in agent["tool_names"]]
+        #     agent["tools"] = [tool if isinstance(tool, Toolkit) else Toolkit(name=tool.name, tools=[tool]) for tool in agent["tools"]]
+        
+        agent_name = self.get_agent_name(agent=agent)
+        if self.has_agent(agent_name=agent_name):
+            return
+        agent_instance = self.create_agent(agent=agent, llm_config=llm_config, **kwargs)
+        self.agents.append(agent_instance)
+        self.agent_states[agent_instance.name] = AgentState.AVAILABLE
+        if agent_instance.name not in self._state_conditions:
+            self._state_conditions[agent_instance.name] = threading.Condition()
+        self.check_agents()
+
+    def add_agents(self, agents: List[Union[str, dict, Agent]], llm_config: Optional[LLMConfig]=None, **kwargs):
+        """
+        add several agents by using self.add_agent().
+        """
+        for agent in agents:
+            self.add_agent(agent=agent, llm_config=llm_config, **kwargs)
+    
+    def add_agents_from_workflow(self, workflow_graph, llm_config: Optional[LLMConfig]=None, **kwargs):
+        """
+        Initialize agents from the nodes of a given WorkFlowGraph and add these agents to self.agents. 
+
+        Args:
+            workflow_graph (WorkFlowGraph): The workflow graph containing nodes with agents information.
+            llm_config (Optional[LLMConfig]): The LLM configuration to be used for the agents.
+            **kwargs (Any): Additional parameters passed to add_agent
+        """
+        from ..workflow.workflow_graph import WorkFlowGraph
+        if not isinstance(workflow_graph, WorkFlowGraph):
+            raise TypeError("workflow_graph must be an instance of WorkFlowGraph")
+        for node in workflow_graph.nodes:
+            if node.agents:
+                for agent in node.agents:
+                    self.add_agent(agent=agent, llm_config=llm_config, **kwargs)
+    
+    def update_agents_from_workflow(self, workflow_graph, llm_config: Optional[LLMConfig]=None, **kwargs):
+        """
+        Update agents from a given WorkFlowGraph.
+
+        Args:
+            workflow_graph (WorkFlowGraph): The workflow graph containing nodes with agents information.
+            llm_config (Optional[LLMConfig]): The LLM configuration to be used for the agents.
+            **kwargs: Additional parameters passed to update_agent
+        """
+        from ..workflow.workflow_graph import WorkFlowGraph
+        if not isinstance(workflow_graph, WorkFlowGraph):
+            raise TypeError("workflow_graph must be an instance of WorkFlowGraph")
+        for node in workflow_graph.nodes:
+            if node.agents:
+                for agent in node.agents:
+                    agent_name = self.get_agent_name(agent=agent)
+                    if self.has_agent(agent_name=agent_name):
+                        # use the llm_config of the existing agent
+                        agent_llm_config = self.get_agent(agent_name).llm_config
+                        self.update_agent(agent=agent, llm_config=agent_llm_config, **kwargs)
+                    else:
+                        self.add_agent(agent=agent, llm_config=llm_config, **kwargs)
+
+    def get_agent(self, agent_name: str, **kwargs) -> Agent:
+        """Retrieve an agent by its name from managed agents.
+        
+        Searches the list of managed agents for an agent with the specified name.
+        
+        Args:
+            agent_name: The name of the agent to retrieve
+            **kwargs (Any): Additional parameters (unused)
+            
+        Returns:
+            The Agent instance with the specified name
+        """
+        for agent in self.agents:
+            if agent.name == agent_name:
+                return agent
+        raise ValueError(f"Agent ``{agent_name}`` does not exists!")
+    
+    def update_agent(self, agent: Union[dict, Agent], llm_config: Optional[LLMConfig]=None, **kwargs):
+        """
+        Update an agent in the manager.
+
+        Args:
+            agent: The agent to be updated, specified as:
+                - Dictionary: Agent specification to update a CustomizeAgent
+                - Agent: Existing Agent instance to update
+            llm_config (Optional[LLMConfig]): The LLM configuration to be used for the agent.
+        """
+        agent_name = self.get_agent_name(agent=agent)
+        self.remove_agent(agent_name=agent_name)
+        self.add_agent(agent=agent, llm_config=llm_config, **kwargs)
+    
+    @atomic_method
+    def remove_agent(self, agent_name: str, remove_from_storage: bool=False, **kwargs):
+        """
+        Remove an agent from the manager and optionally from storage.
+        
+        Args:
+            agent_name: The name of the agent to remove
+            remove_from_storage: If True, also remove the agent from storage
+            **kwargs (Any): Additional parameters passed to storage_handler.remove_agent
+        """
+        self.agents = [agent for agent in self.agents if agent.name != agent_name]
+        self.agent_states.pop(agent_name, None)
+        self._state_conditions.pop(agent_name, None) 
+        if remove_from_storage:
+            self.storage_handler.remove_agent(agent_name=agent_name, **kwargs)
+        self.check_agents()
+
+    def get_agent_state(self, agent_name: str) -> AgentState:
+        """
+        Get the state of a specific agent by its name.
+
+        Args:
+            agent_name: The name of the agent.
+
+        Returns:
+            AgentState: The current state of the agent.
+        """
+        return self.agent_states[agent_name]
+    
+    @atomic_method
+    def set_agent_state(self, agent_name: str, new_state: AgentState) -> bool:
+        """
+        Changes an agent's state and notifies any threads waiting on that agent's state.
+        Thread-safe operation for coordinating multi-threaded agent execution.
+        
+        Args:
+            agent_name: The name of the agent
+            new_state: The new state to set
+        
+        Returns:
+            True if the state was updated successfully, False otherwise
+        """
+        
+        # if agent_name in self.agent_states and isinstance(new_state, AgentState):
+        #     # self.agent_states[agent_name] = new_state
+        #     with self._state_conditions[agent_name]:
+        #         self.agent_states[agent_name] = new_state
+        #         self._state_conditions[agent_name].notify_all()
+        #     self.check_agents()
+        #     return True
+        # else:
+        #     return False
+        if agent_name in self.agent_states and isinstance(new_state, AgentState):
+            if agent_name not in self._state_conditions:
+                self._state_conditions[agent_name] = threading.Condition()
+            with self._state_conditions[agent_name]:
+                self.agent_states[agent_name] = new_state
+                self._state_conditions[agent_name].notify_all()
+            return True
+        return False
+
+    def get_all_agent_states(self) -> Dict[str, AgentState]:
+        """Get the states of all managed agents.
+
+        Returns:
+            Dict[str, AgentState]: A dictionary mapping agent names to their states.
+        """
+        return self.agent_states
+    
+    @atomic_method
+    def save_all_agents(self, **kwargs):
+        """Save all managed agents to persistent storage.
+                
+        Args:
+            **kwargs (Any): Additional parameters passed to the storage handler
+        """
+        pass 
+    
+    @atomic_method
+    def clear_agents(self):
+        """
+        Remove all agents from the manager.
+        """
+        self.agents = [] 
+        self.agent_states = {}
+        self._state_conditions = {}
+        self.check_agents()
+
+    def wait_for_agent_available(self, agent_name: str, timeout: Optional[float] = None) -> bool:
+        """Wait for an agent to be available.
+        
+        Args:
+            agent_name: The name of the agent to wait for
+            timeout: Maximum time to wait in seconds, or None to wait indefinitely
+            
+        Returns:
+            True if the agent became available, False if timed out
+        """
+        if agent_name not in self._state_conditions:
+            self._state_conditions[agent_name] = threading.Condition()
+        condition = self._state_conditions[agent_name]
+
+        with condition:
+            return condition.wait_for(
+                lambda: self.agent_states.get(agent_name) == AgentState.AVAILABLE,
+                timeout=timeout
+            )
+
+    def copy(self) -> "AgentManager":
+        """
+        Create a shallow copy of the AgentManager.
+        """
+        return AgentManager(agents=self.agents, storage_handler=self.storage_handler)

evoagentx/agents/customize_agent.py

@@ -0,0 +1,522 @@
+import json
+import inspect
+from pydantic import create_model, Field
+from typing import Optional, Callable, Type, List, Any, Union, Dict
+
+from .agent import Agent
+from ..core.logging import logger
+from ..core.registry import MODULE_REGISTRY, PARSE_FUNCTION_REGISTRY
+from ..core.message import Message, MessageType
+from ..models.model_configs import LLMConfig 
+from ..models.base_model import PARSER_VALID_MODE
+from ..prompts.utils import DEFAULT_SYSTEM_PROMPT
+from ..prompts.template import PromptTemplate
+from ..actions.action import Action, ActionOutput
+from ..utils.utils import generate_dynamic_class_name, make_parent_folder
+from ..actions.customize_action import CustomizeAction
+from ..actions.action import ActionInput
+from ..tools.tool import Toolkit, Tool
+
+
+class CustomizeAgent(Agent):
+
+    """
+    CustomizeAgent provides a flexible framework for creating specialized LLM-powered agents without 
+    writing custom code. It enables the creation of agents with well-defined inputs and outputs, 
+    custom prompt templates, and configurable parsing strategies. 
+    
+    Attributes:
+        name (str): The name of the agent.
+        description (str): A description of the agent's purpose and capabilities.
+        prompt_template (PromptTemplate, optional): The prompt template that will be used for the agent's primary action. 
+        prompt (str, optional): The prompt template that will be used for the agent's primary action.
+            Should contain placeholders in the format `{input_name}` for each input parameter.
+        llm_config (LLMConfig, optional): Configuration for the language model.
+        inputs (List[dict], optional): List of input specifications, where each dict (e.g., `{"name": str, "type": str, "description": str, ["required": bool]}`) contains:
+            - name (str): Name of the input parameter
+            - type (str): Type of the input
+            - description (str): Description of what the input represents
+            - required (bool, optional): Whether this input is required (default: True)
+        outputs (List[dict], optional): List of output specifications, where each dict (e.g., `{"name": str, "type": str, "description": str, ["required": bool]}`) contains:
+            - name (str): Name of the output field
+            - type (str): Type of the output
+            - description (str): Description of what the output represents
+            - required (bool, optional): Whether this output is required (default: True)
+        system_prompt (str, optional): The system prompt for the LLM. Defaults to DEFAULT_SYSTEM_PROMPT.
+        output_parser (Type[ActionOutput], optional): A custom class for parsing the LLM's output.
+            Must be a subclass of ActionOutput.
+        parse_mode (str, optional): Mode for parsing LLM output. Options are:
+            - "title": Parse outputs using section titles (default)
+            - "str": Parse as plain text
+            - "json": Parse as JSON
+            - "xml": Parse as XML
+            - "custom": Use a custom parsing function
+        parse_func (Callable, optional): Custom function for parsing LLM output when parse_mode is "custom".
+            Must accept a "content" parameter and return a dictionary.
+        title_format (str, optional): Format string for title parsing mode with {title} placeholder.
+            Default is "## {title}".
+        tools (list[Toolkit], optional): List of tools to be used by the agent.
+        max_tool_calls (int, optional): Maximum number of tool calls. Defaults to 5. 
+        custom_output_format (str, optional): Specify the output format. Only used when `prompt_template` is used. 
+            If not provided, the output format will be constructed from the `outputs` specification and `parse_mode`. 
+    """
+    def __init__(
+        self, 
+        name: str, 
+        description: str, 
+        prompt: Optional[str] = None, 
+        prompt_template: Optional[PromptTemplate] = None, 
+        llm_config: Optional[LLMConfig] = None, 
+        inputs: Optional[List[dict]] = None, 
+        outputs: Optional[List[dict]] = None, 
+        system_prompt: Optional[str] = None,
+        output_parser: Optional[Type[ActionOutput]] = None, 
+        parse_mode: Optional[str] = "title", 
+        parse_func: Optional[Callable] = None, 
+        title_format: Optional[str] = None, 
+        tools: Optional[List[Union[Toolkit, Tool]]] = None,
+        max_tool_calls: Optional[int] = 5,
+        custom_output_format: Optional[str] = None, 
+        **kwargs
+    ):
+        system_prompt = system_prompt or DEFAULT_SYSTEM_PROMPT
+        inputs = inputs or [] 
+        outputs = outputs or [] 
+        if tools is not None:
+            raw_tool_map = {tool.name: tool for tool in tools}
+            tools = [tool if isinstance(tool, Toolkit) else Toolkit(name=tool.name, tools=[tool]) for tool in tools]
+        else:
+            raw_tool_map = None
+
+        if prompt is not None and prompt_template is not None:
+            logger.warning("Both `prompt` and `prompt_template` are provided in `CustomizeAgent`. `prompt_template` will be used.")
+            prompt = None 
+
+        if isinstance(parse_func, str):
+            if not PARSE_FUNCTION_REGISTRY.has_function(parse_func):
+                raise ValueError(f"parse function `{parse_func}` is not registered! To instantiate a CustomizeAgent from a file, you should use decorator `@register_parse_function` to register the parse function.")
+            parse_func = PARSE_FUNCTION_REGISTRY.get_function(parse_func)
+        
+        if isinstance(output_parser, str):
+            output_parser = MODULE_REGISTRY.get_module(output_parser)
+        
+        # set default title format 
+        if parse_mode == "title" and title_format is None:
+            title_format = "## {title}"
+
+        # validate the data 
+        self.validate_data(
+            prompt = prompt, 
+            prompt_template = prompt_template, 
+            inputs = inputs, 
+            outputs = outputs, 
+            output_parser = output_parser, 
+            parse_mode = parse_mode, 
+            parse_func = parse_func, 
+            title_format = title_format
+        )
+
+        customize_action = self.create_customize_action(
+            name=name, 
+            desc=description, 
+            prompt=prompt, 
+            prompt_template=prompt_template, 
+            inputs=inputs, 
+            outputs=outputs, 
+            parse_mode=parse_mode, 
+            parse_func=parse_func,
+            output_parser=output_parser,
+            title_format=title_format,
+            custom_output_format=custom_output_format ,
+            tools=tools,
+            max_tool_calls=max_tool_calls
+        )
+        super().__init__(
+            name=name, 
+            description=description, 
+            llm_config=llm_config, 
+            system_prompt=system_prompt, 
+            actions=[customize_action], 
+            **kwargs
+        )
+        self._store_inputs_outputs_info(inputs, outputs, raw_tool_map)
+        self.output_parser = output_parser 
+        self.parse_mode = parse_mode 
+        self.parse_func = parse_func 
+        self.title_format = title_format
+        self.tools = tools
+        self.max_tool_calls = max_tool_calls
+        self.custom_output_format = custom_output_format
+
+    def _add_tools(self, tools: List[Toolkit]):
+        self.get_action(self.customize_action_name).add_tools(tools)
+
+    @property
+    def customize_action_name(self) -> str:
+        """
+        Get the name of the primary custom action for this agent.
+        
+        Returns:
+            The name of the primary custom action
+        """
+        for action in self.actions:
+            if action.name != self.cext_action_name:
+                return action.name
+        raise ValueError("Couldn't find the customize action name!")
+
+    @property
+    def action(self) -> Action:
+        """
+        Get the primary custom action for this agent.
+        
+        Returns:
+            The primary custom action
+        """
+        return self.get_action(self.customize_action_name) 
+    
+    @property
+    def prompt(self) -> str:
+        """
+        Get the prompt for the primary custom action.
+        
+        Returns:
+            The prompt for the primary custom action
+        """
+        return self.action.prompt
+    
+    @property
+    def prompt_template(self) -> PromptTemplate:
+        """
+        Get the prompt template for the primary custom action.
+        
+        Returns:
+            The prompt template for the primary custom action
+        """
+        return self.action.prompt_template
+    
+    def validate_data(self, prompt: str, prompt_template: PromptTemplate, inputs: List[dict], outputs: List[dict], output_parser: Type[ActionOutput], parse_mode: str, parse_func: Callable, title_format: str):
+
+        # check if the prompt is provided
+        if prompt is None and prompt_template is None:
+            raise ValueError("`prompt` or `prompt_template` is required when creating a CustomizeAgent.")
+        
+        # check if all the inputs are in the prompt (only used when prompt_template is not provided)
+        if prompt_template is None and inputs:
+            all_input_names = [input_item["name"] for input_item in inputs]
+            inputs_names_not_in_prompt = [name for name in all_input_names if f'{{{name}}}' not in prompt]
+            if inputs_names_not_in_prompt:
+                raise KeyError(f"The following inputs are not found in the prompt: {inputs_names_not_in_prompt}.") 
+        
+        # check if the output_parser is valid 
+        if output_parser is not None:
+            self._check_output_parser(outputs, output_parser)
+        
+        # check the parse_mode, parse_func, and title_format
+        if parse_mode not in PARSER_VALID_MODE:
+            raise ValueError(f"'{parse_mode}' is an invalid value for `parse_mode`. Available choices: {PARSER_VALID_MODE}.")
+        
+        if parse_mode == "custom":
+            if parse_func is None:
+                raise ValueError("`parse_func` (a callable function with an input argument `content`) must be provided when `parse_mode` is 'custom'.")
+        
+        if parse_func is not None:
+            if not callable(parse_func):
+                raise ValueError("`parse_func` must be a callable function with an input argument `content`.")
+            signature = inspect.signature(parse_func)
+            if "content" not in signature.parameters:
+                raise ValueError("`parse_func` must have an input argument `content`.")
+            if not PARSE_FUNCTION_REGISTRY.has_function(parse_func.__name__):
+                logger.warning(
+                    f"parse function `{parse_func.__name__}` is not registered. This can cause issues when loading the agent from a file. "
+                    f"It is recommended to register the parse function using `register_parse_function`:\n"
+                    f"from evoagentx.core.registry import register_parse_function\n"
+                    f"@register_parse_function\n"
+                    f"def {parse_func.__name__}(content: str) -> dict:\n"
+                    r"    return {'output_name': output_value}" 
+                )
+
+        if title_format is not None:
+            if parse_mode != "title":
+                logger.warning(f"`title_format` will not be used because `parse_mode` is '{parse_mode}', not 'title'. Set `parse_mode='title'` to use title formatting.")
+            if r'{title}' not in title_format:
+                raise ValueError(r"`title_format` must contain the placeholder `{title}`.")
+            
+    def create_customize_action(
+        self, 
+        name: str, 
+        desc: str, 
+        prompt: str, 
+        prompt_template: PromptTemplate, 
+        inputs: List[dict], 
+        outputs: List[dict], 
+        parse_mode: str, 
+        parse_func: Optional[Callable] = None,
+        output_parser: Optional[ActionOutput] = None,
+        title_format: Optional[str] = "## {title}",
+        custom_output_format: Optional[str] = None,
+        tools: Optional[List[Toolkit]] = None,
+        max_tool_calls: Optional[int] = 5
+    ) -> Action:
+        """Create a custom action based on the provided specifications.
+        
+        This method dynamically generates an Action class and instance with:
+        - Input parameters defined by the inputs specification
+        - Output format defined by the outputs specification
+        - Custom execution logic using the customize_action_execute function
+        - If tools is provided, returns a CustomizeAction action instead
+        
+        Args:
+            name: Base name for the action
+            desc: Description of the action
+            prompt: Prompt template for the action
+            prompt_template: Prompt template for the action
+            inputs: List of input field specifications
+            outputs: List of output field specifications
+            parse_mode: Mode to use for parsing LLM output
+            parse_func: Optional custom parsing function
+            output_parser: Optional custom output parser class
+            tools: Optional list of tools
+            
+        Returns:
+            A newly created Action instance
+        """
+        assert prompt is not None or prompt_template is not None, "must provide `prompt` or `prompt_template` when creating CustomizeAgent"
+
+        # create the action input type
+        action_input_fields = {}
+        for field in inputs:
+            required = field.get("required", True)
+            if required:
+                action_input_fields[field["name"]] = (str, Field(description=field["description"]))
+            else:
+                action_input_fields[field["name"]] = (Optional[str], Field(default=None, description=field["description"]))
+
+        action_input_type = create_model(
+            self._get_unique_class_name(
+                generate_dynamic_class_name(name+" action_input")
+            ),
+            **action_input_fields, 
+            __base__=ActionInput
+        )
+        
+        # create the action output type
+        if output_parser is None:
+            action_output_fields = {}
+            for field in outputs:
+                required = field.get("required", True)
+                if required:
+                    action_output_fields[field["name"]] = (Any, Field(description=field["description"]))
+                else:
+                    action_output_fields[field["name"]] = (Optional[Any], Field(default=None, description=field["description"]))
+            action_output_type = create_model(
+                self._get_unique_class_name(
+                    generate_dynamic_class_name(name+" action_output")
+                ),
+                **action_output_fields, 
+                __base__=ActionOutput,
+                # get_content_data=customize_get_content_data,
+                # to_str=customize_to_str
+            )
+        else:
+            # self._check_output_parser(outputs, output_parser)
+            action_output_type = output_parser
+        
+        action_cls_name = self._get_unique_class_name(
+            generate_dynamic_class_name(name+" action")
+        )
+
+        # Create CustomizeAction-based action with parsing properties only
+        customize_action_cls = create_model(
+            action_cls_name,
+            __base__=CustomizeAction
+        )
+
+        customize_action = customize_action_cls(
+            name=action_cls_name,
+            description=desc, 
+            prompt=prompt,
+            prompt_template=prompt_template, 
+            inputs_format=action_input_type,
+            outputs_format=action_output_type,
+            parse_mode=parse_mode,
+            parse_func=parse_func,
+            title_format=title_format,
+            custom_output_format=custom_output_format,
+            max_tool_try=max_tool_calls,
+            tools=tools
+        )
+
+        return customize_action
+    
+    def _check_output_parser(self, outputs: List[dict], output_parser: Type[ActionOutput]):
+
+        if output_parser is not None:
+            if not isinstance(output_parser, type):
+                raise TypeError(f"output_parser must be a class, but got {type(output_parser).__name__}")
+            if not issubclass(output_parser, ActionOutput):
+                raise ValueError(f"`output_parser` must be a class and a subclass of `ActionOutput`, but got `{output_parser.__name__}`.")
+        
+        # check if the output parser is compatible with the outputs
+        output_parser_fields = output_parser.get_attrs()
+        all_output_names = [output_item["name"] for output_item in outputs]
+        for field in output_parser_fields:
+            if field not in all_output_names:
+                raise ValueError(
+                    f"The output parser `{output_parser.__name__}` is not compatible with the `outputs`.\n"
+                    f"The output parser fields: {output_parser_fields}.\n"
+                    f"The outputs: {all_output_names}.\n"
+                    f"All the fields in the output parser must be present in the outputs." 
+                )
+    
+    def _store_inputs_outputs_info(self, inputs: List[dict], outputs: List[dict], tool_map: Dict[str, Union[Toolkit, Tool]]):
+
+        self._action_input_types, self._action_input_required = {}, {} 
+        for field in inputs:
+            required = field.get("required", True)
+            self._action_input_types[field["name"]] = field["type"]
+            self._action_input_required[field["name"]] = required
+        self._action_output_types, self._action_output_required = {}, {}
+        for field in outputs:
+            required = field.get("required", True)
+            self._action_output_types[field["name"]] = field["type"]
+            self._action_output_required[field["name"]] = required
+        self._raw_tool_map = tool_map
+    
+    def __call__(self, inputs: dict = None, return_msg_type: MessageType = MessageType.UNKNOWN, **kwargs) -> Message:
+        """
+        Call the customize action.
+
+        Args:
+            inputs (dict): The inputs to the customize action.
+            **kwargs (Any): Additional keyword arguments.
+
+        Returns:
+            ActionOutput: The output of the customize action.
+        """
+        # return self.execute(action_name=self.customize_action_name, action_input_data=inputs, **kwargs) 
+        inputs = inputs or {} 
+        return super().__call__(action_name=self.customize_action_name, action_input_data=inputs, return_msg_type=return_msg_type, **kwargs)
+    
+    def get_customize_agent_info(self) -> dict:
+        """
+        Get the information of the customize agent.
+        """
+        customize_action = self.get_action(self.customize_action_name)
+        action_input_params = customize_action.inputs_format.get_attrs()
+        action_output_params = customize_action.outputs_format.get_attrs()
+        
+        config = {
+            "class_name": "CustomizeAgent",
+            "name": self.name,
+            "description": self.description,
+            "prompt": customize_action.prompt,
+            "prompt_template": customize_action.prompt_template.to_dict() if customize_action.prompt_template is not None else None, 
+            # "llm_config": self.llm_config.to_dict(exclude_none=True),
+            "inputs": [
+                {
+                    "name": field,
+                    "type": self._action_input_types[field],
+                    "description": field_info.description,
+                    "required": self._action_input_required[field]
+                }
+                for field, field_info in customize_action.inputs_format.model_fields.items() if field in action_input_params
+            ],
+            "outputs": [
+                {
+                    "name": field,
+                    "type": self._action_output_types[field],
+                    "description": field_info.description,
+                    "required": self._action_output_required[field]
+                }
+                for field, field_info in customize_action.outputs_format.model_fields.items() if field in action_output_params
+            ],
+            "system_prompt": self.system_prompt,
+            "output_parser": self.output_parser.__name__ if self.output_parser is not None else None,
+            "parse_mode": self.parse_mode,
+            "parse_func": self.parse_func.__name__ if self.parse_func is not None else None,
+            "title_format": self.title_format,
+            "tool_names": [tool.name for tool in customize_action.tools] if customize_action.tools else [],
+            "max_tool_calls": self.max_tool_calls,
+            "custom_output_format": self.custom_output_format
+        }
+        return config
+    
+    @classmethod
+    def load_module(cls, path: str, llm_config: LLMConfig = None, tools: List[Union[Toolkit, Tool]] = None, **kwargs) -> "CustomizeAgent":
+        """
+        load the agent from local storage. Must provide `llm_config` when loading the agent from local storage. 
+            If tools is provided, tool_names must also be provided. 
+
+        Args:
+            path: The path of the file
+            llm_config: The LLMConfig instance
+            tool_names: List of tool names to be used by the agent. If provided,
+            tool_dict: Dictionary mapping tool names to Tool instances. Required when tool_names is provided.
+
+        Returns:
+            CustomizeAgent: The loaded agent instance
+        """
+        match_dict = {}
+        agent = super().load_module(path=path, llm_config=llm_config, **kwargs)
+        if tools:
+            match_dict = {tool.name:tool for tool in tools}
+        if agent.get("tool_names", None):
+            assert tools is not None, "must provide `tools: List[Union[Toolkit, Tool]]` when using `load_module` or `from_file` to load the agent from local storage and `tool_names` is not None or empty"
+            added_tools = [match_dict[tool_name] for tool_name in agent["tool_names"]]
+            agent["tools"] = [tool if isinstance(tool, Toolkit) else Toolkit(name=tool.name, tools=[tool]) for tool in added_tools]
+        return agent 
+    
+    def save_module(self, path: str, ignore: List[str] = [], **kwargs)-> str:
+        """Save the customize agent's configuration to a JSON file.
+        
+        Args:
+            path: File path where the configuration should be saved
+            ignore: List of keys to exclude from the saved configuration
+            **kwargs (Any): Additional parameters for the save operation
+            
+        Returns:
+            The path where the configuration was saved
+        """
+        config = self.get_customize_agent_info()
+
+        for ignore_key in ignore:
+            config.pop(ignore_key, None)
+        
+        # Save to JSON file
+        make_parent_folder(path)
+        with open(path, 'w', encoding='utf-8') as f:
+            json.dump(config, f, indent=4, ensure_ascii=False)
+
+        return path
+    
+    def _get_unique_class_name(self, candidate_name: str) -> str:
+        """
+        Get a unique class name by checking if it already exists in the registry.
+        If it does, append "Vx" to make it unique.
+        """
+        if not MODULE_REGISTRY.has_module(candidate_name):
+            return candidate_name 
+        
+        i = 1 
+        while True:
+            unique_name = f"{candidate_name}V{i}"
+            if not MODULE_REGISTRY.has_module(unique_name):
+                break
+            i += 1 
+        return unique_name 
+    
+    def get_config(self) -> dict:
+        """
+        Get a dictionary containing all necessary configuration to recreate this agent.
+        
+        Returns:
+            dict: A configuration dictionary that can be used to initialize a new Agent instance
+            with the same properties as this one.
+        """
+        config = self.get_customize_agent_info()
+        config["llm_config"] = self.llm_config.to_dict()
+        tool_names = config.pop("tool_names", None)
+        if tool_names:
+            config["tools"] = [self._raw_tool_map[name] for name in tool_names]
+        return config
+    

evoagentx/agents/long_term_memory_agent.py

@@ -0,0 +1,491 @@
+import json
+import asyncio
+from uuid import uuid4
+from pydantic import Field
+from datetime import datetime
+from typing import Optional, List, Tuple, Dict, Union
+
+from evoagentx.agents import Agent
+from evoagentx.core.parser import Parser
+from evoagentx.models import BaseLLM
+from evoagentx.core.logging import logger
+from evoagentx.models import OpenAILLMConfig
+from evoagentx.storages.base import StorageHandler
+from evoagentx.core.message import Message, MessageType
+from evoagentx.memory.memory_manager import MemoryManager
+from evoagentx.memory.long_term_memory import LongTermMemory
+from evoagentx.actions.action import Action, ActionInput, ActionOutput
+from evoagentx.rag.rag_config import RAGConfig
+
+
+class MemoryActionInput(ActionInput):
+    user_prompt: str = Field(description="The user's input prompt")
+    conversation_id: Optional[str] = Field(default=None, description="ID for tracking conversation")
+    top_k: Optional[int] = Field(default=5, description="Number of memory results to retrieve")
+    metadata_filters: Optional[Dict] = Field(default=None, description="Filters for memory retrieval")
+
+
+class MemoryActionOutput(ActionOutput):
+    response: str = Field(description="The agent's response based on memory and prompt")
+
+
+class MemoryAction(Action):
+    def __init__(
+        self,
+        name: str = "MemoryAction",
+        description: str = "Action that processes user input with long-term memory context",
+        prompt: str = "Based on the following context and user prompt, provide a relevant response:\n\nContext: {context}\n\nUser Prompt: {user_prompt}\n\n",
+        inputs_format: ActionInput = None,
+        outputs_format: ActionOutput = None,
+        **kwargs
+    ):
+        inputs_format = inputs_format or MemoryActionInput
+        outputs_format = outputs_format or MemoryActionOutput
+        super().__init__(
+            name=name,
+            description=description,
+            prompt=prompt,
+            inputs_format=inputs_format,
+            outputs_format=outputs_format,
+            **kwargs
+        )
+
+    def execute(self, llm: BaseLLM | None = None, 
+                inputs: Dict | None = None, 
+                sys_msg: str | None = None, 
+                return_prompt: bool = False, 
+                memory_manager: Optional[MemoryManager] = None,
+                **kwargs
+    ) -> Parser | Tuple[Parser | str] | None:
+        return asyncio.run(self.async_execute(llm, inputs, sys_msg, return_prompt, memory_manager, **kwargs))
+
+    async def async_execute(
+        self,
+        llm: Optional["BaseLLM"] = None,
+        inputs: Optional[Dict] = None,
+        sys_msg: Optional[str] = None,
+        return_prompt: bool = False,
+        memory_manager: Optional[MemoryManager] = None,
+        **kwargs
+    ) -> Union[MemoryActionOutput, tuple]:
+        if not memory_manager:
+            logger.error("MemoryManager is required for MemoryAction execution")
+            raise ValueError("MemoryManager is required for MemoryAction")
+
+        action_input = self.inputs_format(**inputs)
+        user_prompt = action_input.user_prompt
+        conversation_id = action_input.conversation_id
+        if not conversation_id:
+            conversation_id = str(uuid4())
+            logger.warning("No conversation_id provided; generated a new UUID4 for this session")
+        top_k = action_input.top_k
+        metadata_filters = action_input.metadata_filters
+
+        message = await memory_manager.create_conversation_message(
+            user_prompt=user_prompt,
+            conversation_id=conversation_id,
+            top_k=top_k,
+            metadata_filters=metadata_filters
+        )
+
+        action_input_attrs = self.inputs_format.get_attrs()
+        action_input_data = {attr: getattr(action_input, attr, "undefined") for attr in action_input_attrs}
+        action_input_data["context"] = message.content
+        prompt = self.prompt.format(**action_input_data)
+        logger.info(f"The New Created Message by LongTermMemory:\n\n{prompt}")
+
+        output = await llm.async_generate(
+            prompt=prompt,
+            system_message=sys_msg,
+            parser=self.outputs_format,
+            parse_mode='str'
+        )
+        
+        response_message = Message(
+            content=output.content,
+            msg_type=MessageType.RESPONSE,
+            timestamp=datetime.now().isoformat(),
+            conversation_id=conversation_id,
+            memory_ids=message.memory_ids
+        )
+        memory_ids = await memory_manager.handle_memory(
+            action="add",
+            data=response_message,
+        )
+
+        # Prepare the final output
+        final_output = self.outputs_format(
+            response=output.content,
+            memory_ids=memory_ids
+        )
+
+        if return_prompt:
+            return final_output, prompt
+        return final_output
+
+
+class MemoryAgent(Agent):
+    memory_manager: Optional[MemoryManager] = Field(default=None, description="Manager for long-term memory operations")
+    inputs: List[Dict] = Field(default_factory=list, description="Input specifications for the memory action")
+    outputs: List[Dict] = Field(default_factory=list, description="Output specifications for the memory action")
+
+    def __init__(
+        self,
+        name: str = "MemoryAgent",
+        description: str = "An agent that uses long-term memory to provide context-aware responses",
+        inputs: Optional[List[Dict]] = None,
+        outputs: Optional[List[Dict]] = None,
+        llm_config: Optional[OpenAILLMConfig] = None,
+        storage_handler: Optional[StorageHandler] = None,
+        rag_config: Optional[RAGConfig] = None,
+        conversation_id: Optional[str] = None,
+        system_prompt: Optional[str] = None,
+        prompt: str = "Based on the following context and user prompt, provide a relevant response:\n\nContext: {context}\n\nUser Prompt: {user_prompt}",
+        **kwargs
+    ):
+        # Define inputs and outputs inspired by CustomizeAgent
+        inputs = inputs or []
+        outputs = outputs or []
+
+        # Initialize base Agent with provided parameters
+        super().__init__(
+            name=name,
+            description=description,
+            llm_config=llm_config,
+            system_prompt=system_prompt,
+            storage_handler=storage_handler,
+            inputs=inputs,
+            outputs=outputs,
+            **kwargs
+        )
+
+        self.long_term_memory = LongTermMemory(
+            storage_handler=storage_handler,
+            rag_config=rag_config,
+            default_corpus_id=conversation_id
+        )
+        self.memory_manager = MemoryManager(
+            memory=self.long_term_memory,
+            llm=llm_config.get_llm() if llm_config else None,
+            use_llm_management=True
+        )
+
+        # Initialize inputs and outputs
+        self.inputs = inputs
+        self.outputs = outputs
+
+        # Initialize actions list and add MemoryAction
+        self.actions = []
+        self._action_map = {}
+        memory_action = MemoryAction(
+            name="MemoryAction",
+            description="Action that processes user input with long-term memory context",
+            prompt=prompt,
+            inputs_format=MemoryActionInput,
+            outputs_format=MemoryActionOutput
+        )
+        self.add_action(memory_action)
+
+    def _create_output_message(
+        self,
+        action_output,
+        action_name: str,
+        action_input_data: Optional[Dict],
+        prompt: str,
+        return_msg_type: MessageType = MessageType.RESPONSE,
+        **kwargs
+    ) -> Message:
+        msg = super()._create_output_message(
+            action_output=action_output,
+            action_name=action_name,
+            action_input_data=action_input_data,
+            prompt=prompt,
+            return_msg_type=return_msg_type,
+            **kwargs
+        )
+
+        if action_input_data and "user_prompt" in action_input_data:
+            user_msg = Message(
+                content=action_input_data["user_prompt"],
+                msg_type=MessageType.REQUEST,
+                conversation_id=msg.conversation_id
+            )
+            asyncio.create_task(self.memory_manager.handle_memory(action="add", data=user_msg))
+
+        response_msg = Message(
+            content=action_output.response if hasattr(action_output, "response") else str(action_output),
+            msg_type=MessageType.RESPONSE,
+            conversation_id=msg.conversation_id
+        )
+        asyncio.create_task(self.memory_manager.handle_memory(action="add", data=response_msg))
+
+        return msg
+
+    async def async_execute(
+        self,
+        action_name: str,
+        msgs: Optional[List[Message]] = None,
+        action_input_data: Optional[Dict] = None,
+        return_msg_type: Optional[MessageType] = MessageType.RESPONSE,
+        return_action_input_data: Optional[bool] = False,
+        **kwargs
+    ) -> Union[Message, Tuple[Message, Dict]]:
+        """
+        Execute an action asynchronously with memory management.
+
+        Args:
+            action_name: Name of the action to execute
+            msgs: Optional list of messages providing context
+            action_input_data: Optional input data for the action
+            return_msg_type: Message type for the return message
+            return_action_input_data: Whether to return the action input data
+            **kwargs: Additional parameters
+
+        Returns:
+            Message or tuple: The execution result, optionally with input data
+        """
+        action, action_input_data = self._prepare_execution(
+            action_name=action_name,
+            msgs=msgs,
+            action_input_data=action_input_data,
+            **kwargs
+        )
+
+        # Execute action with memory_manager
+        execution_results = await action.async_execute(
+            llm=self.llm,
+            inputs=action_input_data,
+            sys_msg=self.system_prompt,
+            return_prompt=True,
+            memory_manager=self.memory_manager,
+            **kwargs
+        )
+        action_output, prompt = execution_results
+
+        message = self._create_output_message(
+            action_output=action_output,
+            prompt=prompt,
+            action_name=action_name,
+            return_msg_type=return_msg_type,
+            action_input_data=action_input_data,
+            **kwargs
+        )
+        if return_action_input_data:
+            return message, action_input_data
+        return message
+
+    def execute(
+        self,
+        action_name: str,
+        msgs: Optional[List[Message]] = None,
+        action_input_data: Optional[Dict] = None,
+        return_msg_type: Optional[MessageType] = MessageType.RESPONSE,
+        return_action_input_data: Optional[bool] = False,
+        **kwargs
+    ) -> Union[Message, Tuple[Message, Dict]]:
+        """
+        Execute an action synchronously with memory management.
+
+        Args:
+            action_name: Name of the action to execute
+            msgs: Optional list of messages providing context
+            action_input_data: Optional input data for the action
+            return_msg_type: Message type for the return message
+            return_action_input_data: Whether to return the action input data
+            **kwargs: Additional parameters
+
+        Returns:
+            Message or tuple: The execution result, optionally with input data
+        """
+        action, action_input_data = self._prepare_execution(
+            action_name=action_name,
+            msgs=msgs,
+            action_input_data=action_input_data,
+            **kwargs
+        )
+
+        # Execute action with memory_manager
+        execution_results = action.execute(
+            llm=self.llm,
+            inputs=action_input_data,
+            sys_msg=self.system_prompt,
+            return_prompt=True,
+            memory_manager=self.memory_manager,
+            **kwargs
+        )
+        action_output, prompt = execution_results
+
+        message = self._create_output_message(
+            action_output=action_output,
+            prompt=prompt,
+            action_name=action_name,
+            return_msg_type=return_msg_type,
+            action_input_data=action_input_data,
+            **kwargs
+        )
+        if return_action_input_data:
+            return message, action_input_data
+        return message
+
+    def chat(
+        self,
+        user_prompt: str,
+        *,
+        conversation_id: Optional[str] = None,
+        top_k: Optional[int] = None,
+        metadata_filters: Optional[dict] = None,
+        return_message: bool = True,
+        **kwargs
+    ):
+        action_input_data = {
+            "user_prompt": user_prompt,
+            "conversation_id": conversation_id or self._default_conversation_id(),
+            "top_k": top_k if top_k is not None else 3,
+            "metadata_filters": metadata_filters or {},
+        }
+        msg = self.execute(
+            action_name="MemoryAction",
+            action_input_data=action_input_data,
+            return_msg_type=MessageType.RESPONSE,
+            **kwargs
+        )
+        return msg if return_message else (getattr(msg, "content", None) or str(msg))
+
+
+    async def async_chat(
+        self,
+        user_prompt: str,
+        *,
+        conversation_id: Optional[str] = None,
+        top_k: Optional[int] = None,
+        metadata_filters: Optional[dict] = None,
+        return_message: bool = True,
+        **kwargs
+    ):
+        action_input_data = {
+            "user_prompt": user_prompt,
+            "conversation_id": conversation_id or self._default_conversation_id(),
+            "top_k": top_k if top_k is not None else 3,
+            "metadata_filters": metadata_filters or {},
+        }
+        msg = await self.async_execute(
+            action_name="MemoryAction",
+            action_input_data=action_input_data,
+            return_msg_type=MessageType.RESPONSE,
+            **kwargs
+        )
+        return msg if return_message else (getattr(msg, "content", None) or str(msg))
+
+
+    def _default_conversation_id(self) -> str:
+        """
+        Session scope: By default, a new uuid4() is returned (new session).
+        User/global scope: Reuse LongTermMemory.default_corpus_id (stable namespace).
+        Note: The final ID is still uniformly managed by MemoryAgent._prepare_execution() (which will override based on the scope).
+        """
+        scope = getattr(self, "conversation_scope", "session")
+        if scope == "session":
+            return str(uuid4())
+        return getattr(getattr(self, "long_term_memory", None), "default_corpus_id", None) or "global_corpus"
+    
+    async def interactive_chat(
+        self,
+        conversation_id: Optional[str] = None,
+        top_k: int = 3,
+        metadata_filters: Optional[dict] = None
+    ):
+        """
+        In interactive chat, each round of input will:
+        1. Retrieve from memory
+        2. Generate a response based on historical context
+        3. Write the input/output to long-term memory and refresh the index 
+        """
+        conversation_id = conversation_id or self._default_conversation_id()
+        metadata_filters = metadata_filters or {}
+
+        print("💬 MemoryAgent has been started (type 'exit' to quit)\n")
+
+        while True:
+            user_prompt = input("You: ").strip()
+            if user_prompt.lower() in ["exit", "quit"]:
+                print("🔚 Conversation ended")
+                break
+
+            # Retrieve historical context
+            retrieved_memories = await self.memory_manager.handle_memory(
+                action="search",
+                user_prompt=user_prompt,
+                top_k=top_k,
+                metadata_filters=metadata_filters
+            )
+
+            context_texts = []
+            for msg, _ in retrieved_memories:
+                if hasattr(msg, "content") and msg.content:
+                    context_texts.append(msg.content)
+            context_str = "\n".join(context_texts)
+
+            # if context_str:
+            #     print(f"📖 Retrieved context from memory:\n{context_str}\n")
+
+            # Concatenate the historical context into the user input and invoke async_chat
+            full_prompt = f"Context:\n{context_str}\n\nUser: {user_prompt}" if context_str else user_prompt
+            msg = await self.async_chat(
+                user_prompt=full_prompt,
+                conversation_id=conversation_id,
+                top_k=top_k,
+                metadata_filters=metadata_filters
+            )
+
+            print(f"Agent: {msg.content}\n")
+
+            # Refresh the index to ensure it can be retrieved in the next round
+            if hasattr(self.memory_manager, "handle_memory_flush"):
+                await self.memory_manager.handle_memory_flush()
+            else:
+                await asyncio.sleep(0.1)
+
+
+
+    def save_module(self, path: str, ignore: List[str] = ["llm", "llm_config", "memory_manager"], **kwargs) -> str:
+        """
+        Save the agent's configuration to a JSON file, excluding memory_manager by default.
+
+        Args:
+            path: File path to save the configuration
+            ignore: List of keys to exclude from the saved configuration
+            **kwargs: Additional parameters for saving
+
+        Returns:
+            str: The path where the configuration was saved
+        """
+        return super().save_module(path=path, ignore=ignore, **kwargs)
+
+    @classmethod
+    def from_file(cls, path: str, llm_config: OpenAILLMConfig, storage_handler: Optional[StorageHandler] = None, rag_config: Optional[RAGConfig] = None, **kwargs) -> "MemoryAgent":
+        """
+        Load a MemoryAgent from a JSON configuration file.
+
+        Args:
+            path: Path to the JSON configuration file
+            llm_config: LLM configuration
+            storage_handler: Optional storage handler
+            rag_config: Optional RAG configuration
+            **kwargs: Additional parameters
+
+        Returns:
+            MemoryAgent: The loaded agent instance
+        """
+        with open(path, 'r', encoding='utf-8') as f:
+            config = json.load(f)
+        return cls(
+            name=config.get("name", "MemoryAgent"),
+            description=config.get("description", "An agent that uses long-term memory"),
+            llm_config=llm_config,
+            storage_handler=storage_handler,
+            rag_config=rag_config,
+            system_prompt=config.get("system_prompt"),
+            prompt=config.get("prompt"),
+            use_long_term_memory=config.get("use_long_term_memory", True),
+            **kwargs
+        )

evoagentx/agents/task_planner.py

@@ -0,0 +1,35 @@
+from .agent import Agent
+from ..actions.task_planning import TaskPlanning
+from ..prompts.task_planner import TASK_PLANNER
+
+
+class TaskPlanner(Agent):
+    """An agent responsible for planning and decomposing high-level tasks into smaller sub-tasks.
+    
+    The TaskPlanner agent analyzes complex goals and breaks them down into a structured
+    sequence of smaller, more manageable tasks. It serves as a critical component in the
+    workflow by creating execution plans that other specialized agents can follow.
+    
+    Attributes:
+        name (str): Name of the task planner agent, defaults to the value in TASK_PLANNER
+        description (str): Description of the agent's purpose and capabilities, defaults to the value in TASK_PLANNER
+        system_prompt (str): System prompt guiding the agent's behavior, defaults to the value in TASK_PLANNER
+        actions (List[Action]): List of actions the agent can perform, defaults to [TaskPlanning()]
+    """
+    def __init__(self, **kwargs):
+
+        name = kwargs.pop("name") if "name" in kwargs else TASK_PLANNER["name"]
+        description = kwargs.pop("description") if "description" in kwargs else TASK_PLANNER["description"]
+        system_prompt = kwargs.pop("system_prompt") if "system_prompt" in kwargs else TASK_PLANNER["system_prompt"]
+        actions = kwargs.pop("actions") if "actions" in kwargs else [TaskPlanning()]
+        super().__init__(name=name, description=description, system_prompt=system_prompt, actions=actions, **kwargs)
+    
+    @property
+    def task_planning_action_name(self):
+        """Get the name of the TaskPlanning action associated with this agent.
+        
+        Returns:
+            The name of the TaskPlanning action in this agent's action registry
+        """
+        return self.get_action_name(action_cls=TaskPlanning)
+    

evoagentx/agents/workflow_reviewer.py

@@ -0,0 +1,14 @@
+from typing import Optional, List
+from .agent import Agent
+from ..core.message import Message # MessageType
+
+
+class WorkFlowReviewer(Agent):
+
+    """
+    Placeholder for the Agent that is responsible for reviewing workflow plans and agents.
+    """
+
+    def execute(self, action_name: str, msgs: Optional[List[Message]] = None, action_input_data: Optional[dict] = None, **kwargs) -> Message:
+
+        raise NotImplementedError("WorkflowReviewer is not implemented yet.") 

evoagentx/app/api.py

@@ -0,0 +1,329 @@
+"""
+API routes for EvoAgentX application.
+"""
+from fastapi import APIRouter, Depends, HTTPException, status, BackgroundTasks
+from fastapi.security import OAuth2PasswordRequestForm
+from typing import List, Dict, Any # , Optional
+from fastapi import Response
+
+from datetime import timedelta
+
+from evoagentx.app.config import settings
+from evoagentx.app.schemas import (
+    AgentCreate, AgentUpdate, AgentResponse,
+    WorkflowCreate, WorkflowUpdate, WorkflowResponse,
+    ExecutionCreate, ExecutionResponse,
+    PaginationParams, SearchParams,
+    Token, UserCreate, UserResponse, # UserLogin, 
+)
+from evoagentx.app.services import AgentService, WorkflowService, WorkflowExecutionService
+from evoagentx.app.security import (
+    create_access_token, 
+    authenticate_user, 
+    create_user, 
+    get_current_active_user,
+    get_current_admin_user
+)
+from evoagentx.app.db import Database, ExecutionStatus 
+
+# Create routers for different route groups
+auth_router = APIRouter(prefix=settings.API_PREFIX)
+agents_router = APIRouter(prefix=settings.API_PREFIX)
+workflows_router = APIRouter(prefix=settings.API_PREFIX)
+executions_router = APIRouter(prefix=settings.API_PREFIX)
+system_router = APIRouter(prefix=settings.API_PREFIX)
+
+# Authentication Routes
+@auth_router.post("/auth/register", response_model=UserResponse, tags=["Authentication"])
+async def register_user(user: UserCreate):
+    """Register a new user."""
+    return await create_user(user)
+
+@auth_router.post("/auth/login", response_model=Token, tags=["Authentication"])
+async def login(form_data: OAuth2PasswordRequestForm = Depends()):
+    """Login and return access token."""
+    user = await authenticate_user(form_data.username, form_data.password)
+    if not user:
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail="Incorrect username or password",
+            headers={"WWW-Authenticate": "Bearer"},
+        )
+    
+    access_token_expires = timedelta(minutes=settings.ACCESS_TOKEN_EXPIRE_MINUTES)
+    access_token = create_access_token(
+        subject=user['email'], 
+        expires_delta=access_token_expires
+    )
+    
+    return {
+        "access_token": access_token, 
+        "token_type": "bearer"
+    }
+
+# Agent Routes
+@agents_router.post("/agents", response_model=AgentResponse, tags=["Agents"])
+async def create_agent(
+    agent: AgentCreate, 
+    current_user: Dict[str, Any] = Depends(get_current_active_user)
+):
+    """Create a new agent."""
+    try:
+        created_agent = await AgentService.create_agent(
+            agent, 
+            user_id=str(current_user['_id'])
+        )
+        # Convert the ObjectId to string before creating the response model
+        created_agent["_id"] = str(created_agent["_id"])
+        return AgentResponse(**created_agent)
+    except ValueError as e:
+        raise HTTPException(status_code=400, detail=str(e))
+
+@agents_router.get("/agents/{agent_id}", response_model=AgentResponse, tags=["Agents"])
+async def get_agent(
+    agent_id: str, 
+    current_user: Dict[str, Any] = Depends(get_current_active_user)
+):
+    """Retrieve a specific agent by ID."""
+    agent = await AgentService.get_agent(agent_id)
+    if not agent:
+        raise HTTPException(status_code=404, detail="Agent not found")
+    agent["_id"] = str(agent["_id"])
+    return AgentResponse(**agent)
+
+@agents_router.put("/agents/{agent_id}", response_model=AgentResponse, tags=["Agents"])
+async def update_agent(
+    agent_id: str, 
+    agent_update: AgentUpdate, 
+    current_user: Dict[str, Any] = Depends(get_current_active_user)
+):
+    """Update an existing agent."""
+    try:
+        updated_agent = await AgentService.update_agent(agent_id, agent_update)
+        if not updated_agent:
+            raise HTTPException(status_code=404, detail="Agent not found")
+        updated_agent["_id"] = str(updated_agent["_id"])
+        return AgentResponse(**updated_agent)
+    except ValueError as e:
+        raise HTTPException(status_code=400, detail=str(e))
+
+@agents_router.get("/agents", response_model=List[AgentResponse], tags=["Agents"])
+async def list_agents(
+    pagination: PaginationParams = Depends(),
+    search: SearchParams = Depends(),
+    current_user: Dict[str, Any] = Depends(get_current_active_user)
+):
+    """List agents with optional pagination and search."""
+    agents, total = await AgentService.list_agents(pagination, search)
+    # Convert _id to string for each agent in the list
+    for agent in agents:
+        agent["_id"] = str(agent["_id"])
+    return [AgentResponse(**agent) for agent in agents]
+
+@agents_router.delete("/agents/{agent_id}", status_code=204, tags=["Agents"])
+async def delete_agent(
+    agent_id: str, 
+    current_user: Dict[str, Any] = Depends(get_current_admin_user)
+):
+    """Delete an agent (admin-only)."""
+    try:
+        success = await AgentService.delete_agent(agent_id)
+        if not success:
+            raise HTTPException(status_code=404, detail="Agent not found")
+        return  # With 204, no content is returned
+    except ValueError as e:
+        raise HTTPException(status_code=400, detail=str(e))
+
+
+
+# Workflow Routes
+@workflows_router.post("/workflows", response_model=WorkflowResponse,status_code=201, tags=["Workflows"])
+async def create_workflow(
+    workflow: WorkflowCreate, 
+    current_user: Dict[str, Any] = Depends(get_current_active_user)
+):
+    """Create a new workflow."""
+    try:
+        created_workflow = await WorkflowService.create_workflow(
+            workflow, 
+            user_id=str(current_user['_id'])
+        )
+        # Convert the ObjectId to string for consistency
+        created_workflow["_id"] = str(created_workflow["_id"])
+        return WorkflowResponse(**created_workflow)
+    except ValueError as e:
+        raise HTTPException(status_code=400, detail=str(e))
+
+
+
+@workflows_router.get("/workflows/{workflow_id}", response_model=WorkflowResponse, tags=["Workflows"])
+async def get_workflow(
+    workflow_id: str, 
+    current_user: Dict[str, Any] = Depends(get_current_active_user)
+):
+    """Retrieve a specific workflow by ID."""
+    workflow = await WorkflowService.get_workflow(workflow_id)
+    if not workflow:
+        raise HTTPException(status_code=404, detail="Workflow not found")
+    # Convert ObjectId to string
+    workflow["_id"] = str(workflow["_id"])
+    return WorkflowResponse(**workflow)
+
+@workflows_router.put("/workflows/{workflow_id}", response_model=WorkflowResponse, tags=["Workflows"])
+async def update_workflow(
+    workflow_id: str, 
+    workflow_update: WorkflowUpdate, 
+    current_user: Dict[str, Any] = Depends(get_current_active_user)
+):
+    """Update an existing workflow."""
+    try:
+        updated_workflow = await WorkflowService.update_workflow(workflow_id, workflow_update)
+        if not updated_workflow:
+            raise HTTPException(status_code=404, detail="Workflow not found")
+        
+        updated_workflow["_id"] = str(updated_workflow["_id"])
+        return WorkflowResponse(**updated_workflow)
+    except ValueError as e:
+        raise HTTPException(status_code=400, detail=str(e))
+
+@workflows_router.delete("/workflows/{workflow_id}", status_code=204, tags=["Workflows"])
+async def delete_workflow(
+    workflow_id: str, 
+    current_user: Dict[str, Any] = Depends(get_current_admin_user)
+):
+    """Delete a workflow (admin-only)."""
+    try:
+        success = await WorkflowService.delete_workflow(workflow_id)
+        if not success:
+            raise HTTPException(status_code=404, detail="Workflow not found")
+        return Response(status_code=204)
+    except ValueError as e:
+        raise HTTPException(status_code=400, detail=str(e))
+
+
+@workflows_router.get("/workflows", response_model=List[WorkflowResponse], tags=["Workflows"])
+async def list_workflows(
+    pagination: PaginationParams = Depends(),
+    search: SearchParams = Depends(),
+    current_user: Dict[str, Any] = Depends(get_current_active_user)
+):
+    """List workflows with optional pagination and search."""
+    workflows, total = await WorkflowService.list_workflows(pagination, search)
+    
+    # Convert ObjectId to string for each workflow
+    converted_workflows = [
+        {**workflow, "_id": str(workflow["_id"])}
+        for workflow in workflows
+    ]
+    
+    return [WorkflowResponse(**workflow) for workflow in converted_workflows]
+
+
+# Workflow Execution Routes
+@executions_router.post("/executions", response_model=ExecutionResponse, status_code=202)
+async def create_execution(
+    execution: ExecutionCreate,
+    background_tasks: BackgroundTasks,
+    current_user: Dict[str, Any] = Depends(get_current_active_user)
+):
+    """Create and start a workflow execution."""
+    try:
+        execution_result = await WorkflowExecutionService.create_execution(
+            execution_data=execution,
+            user_id=str(current_user['_id'])
+        )
+        # Convert _id to string for consistency
+        execution_result["_id"] = str(execution_result["_id"])
+        return ExecutionResponse(**execution_result)
+    except ValueError as e:
+        raise HTTPException(status_code=400, detail=str(e))
+
+
+@executions_router.get("/executions/{execution_id}", response_model=ExecutionResponse)
+async def get_execution(
+    execution_id: str,
+    current_user: Dict[str, Any] = Depends(get_current_active_user)
+):
+    """Retrieve a specific workflow execution by ID."""
+    try:
+        execution = await WorkflowExecutionService.get_execution(execution_id)
+        if not execution:
+            raise HTTPException(status_code=404, detail="Execution not found")
+        execution["_id"] = str(execution["_id"])
+        return ExecutionResponse(**execution)
+    except ValueError as e:
+        raise HTTPException(status_code=400, detail=str(e))
+
+
+@executions_router.post("/executions/{execution_id}/stop", response_model=ExecutionResponse)
+async def stop_execution(
+    execution_id: str,
+    current_user: Dict[str, Any] = Depends(get_current_active_user)
+):
+    """Stop (cancel) a workflow execution."""
+    try:
+        updated_execution = await WorkflowExecutionService.update_execution_status(
+            execution_id=execution_id,
+            status=ExecutionStatus.CANCELLED
+        )
+        if not updated_execution:
+            raise HTTPException(status_code=404, detail="Execution not found")
+        # Convert ObjectId to string for consistency
+        updated_execution["_id"] = str(updated_execution["_id"])
+        return ExecutionResponse(**updated_execution)
+    except ValueError as e:
+        raise HTTPException(status_code=400, detail=str(e))
+
+
+@executions_router.get("/executions", response_model=List[ExecutionResponse])
+async def list_executions(
+    pagination: PaginationParams = Depends(),
+    search: SearchParams = Depends(), 
+    current_user: Dict[str, Any] = Depends(get_current_active_user)
+):
+    """List workflow executions with optional pagination and search."""
+    executions, total = await WorkflowExecutionService.list_executions(
+        params=pagination, 
+        search=search
+    )
+    # Convert _id to string for each execution
+    for exec_item in executions:
+        exec_item["_id"] = str(exec_item["_id"])
+    return [ExecutionResponse(**exec_item) for exec_item in executions]
+
+
+@executions_router.get("/executions/{execution_id}/logs", response_model=List[Dict[str, Any]])
+async def get_execution_logs(
+    execution_id: str,
+    pagination: PaginationParams = Depends(),
+    current_user: Dict[str, Any] = Depends(get_current_active_user)
+):
+    """Retrieve logs for a specific execution."""
+    logs, total = await WorkflowExecutionService.get_execution_logs(execution_id, params=pagination)
+    # Convert _id in each log entry to string
+    for log in logs:
+        log["_id"] = str(log["_id"])
+    return logs
+
+# Health Check Route
+@system_router.get("/health", tags=["System"])
+async def health_check():
+    """Simple health check endpoint."""
+    try:
+        # You can add more comprehensive health checks here
+        await Database.db.command('ping')
+        return {
+            "status": "healthy", 
+            "version": "1.0.0"
+        }
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Database connection error: {str(e)}")
+
+# Export the routers
+__all__ = [
+    'auth_router',
+    'agents_router',
+    'workflows_router',
+    'executions_router',
+    'system_router'
+]

evoagentx/app/app.env

@@ -0,0 +1,22 @@
+# .env file
+APP_NAME=EvoAgentX
+DEBUG=True
+API_PREFIX=/api/v1
+HOST=0.0.0.0
+PORT=8000
+
+# MongoDB settings
+MONGODB_URL=mongodb+srv://eax:eax@cluster0.1lkbi0y.mongodb.net/?retryWrites=true&w=majority&appName=Cluster0
+MONGODB_DB_NAME=evoagentx
+
+# JWT Authentication
+SECRET_KEY=your-secret-key
+ACCESS_TOKEN_EXPIRE_MINUTES=30
+ALGORITHM=HS256
+
+# Logging
+LOG_LEVEL=INFO
+
+# CORS settings
+ALLOWED_HOSTS: List[str]
+CORS_ORIGINS: List[str]

evoagentx/app/config.py

@@ -0,0 +1,83 @@
+"""
+Configuration settings for the EvoAgentX application.
+"""
+# import os
+from pydantic import BaseModel, Field, validator
+from pydantic_settings import BaseSettings
+from typing import Optional, Dict, Any, List
+
+class Settings(BaseSettings):
+    # Application settings
+    APP_NAME: str
+    DEBUG: bool
+    API_PREFIX: str
+    HOST: str
+    PORT: int
+    
+    # MongoDB settings
+    MONGODB_URL: str
+    MONGODB_DB_NAME: str
+    
+    # JWT Authentication
+    SECRET_KEY: str
+    ACCESS_TOKEN_EXPIRE_MINUTES: int
+    ALGORITHM: str
+    
+    # Logging configuration
+    LOG_LEVEL: str
+    
+    # Add CORS settings
+    CORS_ORIGINS: List[str] = ["http://localhost:3000", "http://localhost:8000"]
+    CORS_ALLOW_CREDENTIALS: bool = True
+    
+    class Config:
+        env_file = ".env"
+        case_sensitive = True
+        env_delimiter = ","
+
+
+
+# Global settings instance
+settings = Settings()
+
+# Agent and Workflow configuration
+class AgentConfig(BaseModel):
+    """Base configuration for an LLM agent."""
+    model_name: str
+    temperature: float = 0.7
+    max_tokens: int = 2048
+    api_key_env_var: Optional[str] = None
+    system_prompt: Optional[str] = None
+    extra_params: Dict[str, Any] = Field(default_factory=dict)
+    
+    @validator('temperature')
+    def validate_temperature(cls, v):
+        if v < 0 or v > 1:
+            raise ValueError('Temperature must be between 0 and 1')
+        return v
+
+class WorkflowStepConfig(BaseModel):
+    """Configuration for a single step in a workflow."""
+    step_id: str
+    agent_id: str
+    action: str
+    input_mapping: Dict[str, str] = Field(default_factory=dict)
+    output_mapping: Dict[str, str] = Field(default_factory=dict)
+    timeout_seconds: int = 300
+    retry_count: int = 3
+    
+class WorkflowConfig(BaseModel):
+    """Configuration for a workflow composed of agent steps."""
+    name: str
+    description: Optional[str] = None
+    steps: List[WorkflowStepConfig]
+    parallel_execution: bool = False
+    timeout_seconds: int = 3600  # Default to 1 hour total timeout
+
+class ExecutionConfig(BaseModel):
+    """Configuration for a workflow execution."""
+    workflow_id: str
+    input_params: Dict[str, Any] = Field(default_factory=dict)
+    user_id: Optional[str] = None
+    priority: int = 1  # Higher number means higher priority
+    callback_url: Optional[str] = None

evoagentx/app/db.py

@@ -0,0 +1,177 @@
+"""
+Database connection and models for EvoAgentX.
+"""
+# import asyncio
+import logging
+from datetime import datetime
+from enum import Enum
+from typing import Optional, List, Dict, Any # , Union
+from motor.motor_asyncio import AsyncIOMotorClient
+from pymongo import ASCENDING, TEXT
+from pydantic_core import core_schema
+from bson import ObjectId
+from pydantic import GetCoreSchemaHandler
+from pydantic import Field, BaseModel
+from evoagentx.app.config import settings
+
+# Setup logger
+logger = logging.getLogger(__name__)
+
+# Custom PyObjectId for MongoDB ObjectId compatibility with Pydantic
+class PyObjectId(ObjectId):
+    @classmethod
+    def __get_pydantic_core_schema__(cls, source_type, handler: GetCoreSchemaHandler):
+        return core_schema.no_info_after_validator_function(cls.validate, core_schema.str_schema())
+
+    @classmethod
+    def validate(cls, v):
+        if not ObjectId.is_valid(v):
+            raise ValueError("Invalid ObjectId")
+        return ObjectId(v)
+
+# Base model with ObjectId handling
+class MongoBaseModel(BaseModel):
+    id: Optional[PyObjectId] = Field(alias="_id", default=None)
+
+    model_config = {
+        "protected_namespaces": (),
+        "populate_by_name": True,  # Replace `allow_population_by_field_name`
+        "arbitrary_types_allowed": True,  # Keep custom types like ObjectId
+        "json_encoders": {
+            ObjectId: str  # Ensure ObjectId is serialized as a string
+        }
+    }
+
+# Status Enums
+class AgentStatus(str, Enum):
+    CREATED = "created"
+    ACTIVE = "active"
+    INACTIVE = "inactive"
+    ERROR = "error"
+
+class WorkflowStatus(str, Enum):
+    CREATED = "created"
+    RUNNING = "running"
+    COMPLETED = "completed"
+    FAILED = "failed"
+    CANCELLED = "cancelled"
+
+class ExecutionStatus(str, Enum):
+    PENDING = "pending"
+    RUNNING = "running"
+    COMPLETED = "completed"
+    FAILED = "failed"
+    TIMEOUT = "timeout"
+    CANCELLED = "cancelled"
+
+# Database Models
+class Agent(MongoBaseModel):
+    id: str = Field(..., alias="_id")
+    name: str
+    description: Optional[str] = None
+    config: Dict[str, Any]
+    state: Dict[str, Any] = Field(default_factory=dict)
+    runtime_params: Dict[str, Any] = Field(default_factory=dict)
+    status: AgentStatus = AgentStatus.CREATED
+    created_at: datetime = Field(default_factory=datetime.utcnow)
+    updated_at: datetime = Field(default_factory=datetime.utcnow)
+    created_by: Optional[str] = None
+    tags: List[str] = Field(default_factory=list)
+
+class Workflow(MongoBaseModel):
+    id: str = Field(..., alias="_id")
+    name: str
+    description: Optional[str] = None
+    definition: Dict[str, Any]
+    agent_ids: List[str] = Field(default_factory=list)
+    status: WorkflowStatus = WorkflowStatus.CREATED
+    created_at: datetime = Field(default_factory=datetime.utcnow)
+    updated_at: datetime = Field(default_factory=datetime.utcnow)
+    created_by: Optional[str] = None
+    tags: List[str] = Field(default_factory=list)
+    version: int = 1
+
+class ExecutionLog(MongoBaseModel):
+    workflow_id: str
+    execution_id: str
+    step_id: Optional[str] = None
+    agent_id: Optional[str] = None
+    timestamp: datetime = Field(default_factory=datetime.utcnow)
+    level: str = "INFO"
+    message: str
+    details: Dict[str, Any] = Field(default_factory=dict)
+
+class WorkflowExecution(MongoBaseModel):
+    workflow_id: str
+    status: ExecutionStatus = ExecutionStatus.PENDING
+    start_time: Optional[datetime] = None
+    end_time: Optional[datetime] = None
+    input_params: Dict[str, Any] = Field(default_factory=dict)
+    results: Dict[str, Any] = Field(default_factory=dict)
+    created_by: Optional[str] = None
+    step_results: Dict[str, Dict[str, Any]] = Field(default_factory=dict)
+    current_step: Optional[str] = None
+    error_message: Optional[str] = None
+    created_at: datetime = Field(default_factory=datetime.utcnow)
+
+# Database client
+class Database:
+    client: AsyncIOMotorClient = None
+    db = None
+    
+    # Collections
+    agents = None
+    workflows = None
+    executions = None
+    logs = None
+    
+    @classmethod
+    async def connect(cls):
+        """Connect to MongoDB"""
+        logger.info(f"Connecting to MongoDB at {settings.MONGODB_URL}...")
+        cls.client = AsyncIOMotorClient(settings.MONGODB_URL)
+        cls.db = cls.client[settings.MONGODB_DB_NAME]
+        
+        # Set up collections
+        cls.agents = cls.db.agents
+        cls.workflows = cls.db.workflows
+        cls.executions = cls.db.workflow_executions
+        cls.logs = cls.db.execution_logs
+        
+        # Create indexes
+        await cls._create_indexes()
+        
+        logger.info("Connected to MongoDB successfully")
+    
+    @classmethod
+    async def disconnect(cls):
+        """Disconnect from MongoDB"""
+        if cls.client:
+            cls.client.close()
+            logger.info("Disconnected from MongoDB")
+    
+    @classmethod
+    async def _create_indexes(cls):
+        """Create indexes for collections"""
+        # Agent indexes
+        await cls.agents.create_index([("name", ASCENDING)], unique=True)
+        await cls.agents.create_index([("name", TEXT), ("description", TEXT)])
+        await cls.agents.create_index([("created_at", ASCENDING)])
+        await cls.agents.create_index([("tags", ASCENDING)])
+        
+        # Workflow indexes
+        await cls.workflows.create_index([("name", ASCENDING)])
+        await cls.workflows.create_index([("name", TEXT), ("description", TEXT)])
+        await cls.workflows.create_index([("created_at", ASCENDING)])
+        await cls.workflows.create_index([("agent_ids", ASCENDING)])
+        await cls.workflows.create_index([("tags", ASCENDING)])
+        
+        # Execution indexes
+        await cls.executions.create_index([("workflow_id", ASCENDING)])
+        await cls.executions.create_index([("created_at", ASCENDING)])
+        await cls.executions.create_index([("status", ASCENDING)])
+        
+        # Log indexes
+        await cls.logs.create_index([("execution_id", ASCENDING)])
+        await cls.logs.create_index([("timestamp", ASCENDING)])
+        await cls.logs.create_index([("workflow_id", ASCENDING), ("execution_id", ASCENDING)])

evoagentx/app/main.py

@@ -0,0 +1,177 @@
+"""
+Main application entry point for EvoAgentX.
+"""
+import logging
+# import asyncio
+from contextlib import asynccontextmanager
+
+import uvicorn
+from fastapi import FastAPI, Request
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import JSONResponse
+from fastapi.exceptions import RequestValidationError, HTTPException
+
+from evoagentx.app.config import settings
+from evoagentx.app.db import Database
+from evoagentx.app.security import init_users_collection
+from evoagentx.app.api import (
+    auth_router,
+    agents_router,
+    workflows_router,
+    executions_router,
+    system_router
+)
+
+# Configure logging
+logging.basicConfig(
+    level=getattr(logging, settings.LOG_LEVEL.upper()),
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+)
+logger = logging.getLogger(__name__)
+
+# Lifespan context manager for startup and shutdown events
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    """
+    Async context manager to handle application startup and shutdown events.
+    """
+    # Startup tasks
+    try:
+        # Connect to database
+        await Database.connect()
+        
+        # Initialize users collection and create admin user if not exists
+        await init_users_collection()
+        
+        logger.info("Application startup completed successfully")
+        yield
+    except Exception as e:
+        logger.error(f"Error during application startup: {e}")
+        raise
+    finally:
+        # Shutdown tasks
+        try:
+            await Database.disconnect()
+            logger.info("Application shutdown completed successfully")
+        except Exception as e:
+            logger.error(f"Error during application shutdown: {e}")
+
+# Create FastAPI application
+app = FastAPI(
+    title="EvoAgentX API",
+    description="API for EvoAgentX platform",
+    version="1.0.0",
+    lifespan=lifespan,
+    docs_url="/docs",
+    redoc_url="/redoc"
+)
+
+# Configure CORS
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=settings.CORS_ORIGINS,
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+# Include routers
+app.include_router(auth_router)
+app.include_router(agents_router)
+app.include_router(workflows_router)
+app.include_router(executions_router)
+app.include_router(system_router)
+
+# Global exception handlers
+@app.exception_handler(RequestValidationError)
+async def validation_exception_handler(request: Request, exc: RequestValidationError):
+    """
+    Custom validation error handler to provide more detailed error responses.
+    """
+    return JSONResponse(
+        status_code=422,
+        content={
+            "status": "error",
+            "message": "Validation error",
+            "errors": exc.errors()
+        }
+    )
+
+@app.exception_handler(HTTPException)
+async def http_exception_handler(request: Request, exc: HTTPException):
+    """
+    Custom HTTP exception handler to provide consistent error responses.
+    """
+    return JSONResponse(
+        status_code=exc.status_code,
+        content={
+            "status": "error",
+            "message": exc.detail
+        }
+    )
+
+# Root endpoint for health check
+@app.get("/")
+async def root():
+    """
+    Root endpoint for application health check.
+    """
+    return {
+        "app_name": settings.APP_NAME,
+        "status": "healthy",
+        "version": "0.1.0"
+    }
+
+# Workflow logging and monitoring endpoint
+@app.get("/metrics")
+async def get_metrics():
+    """
+    Endpoint to retrieve system metrics and stats.
+    """
+    # Collect metrics from different services
+    try:
+        # Collect agent metrics
+        total_agents = await Database.agents.count_documents({})
+        active_agents = await Database.agents.count_documents({"status": "active"})
+        
+        # Collect workflow metrics
+        total_workflows = await Database.workflows.count_documents({})
+        running_workflows = await Database.workflows.count_documents({"status": "running"})
+        
+        # Collect execution metrics
+        total_executions = await Database.executions.count_documents({})
+        failed_executions = await Database.executions.count_documents({"status": "failed"})
+        
+        return {
+            "agents": {
+                "total": total_agents,
+                "active": active_agents
+            },
+            "workflows": {
+                "total": total_workflows,
+                "running": running_workflows
+            },
+            "executions": {
+                "total": total_executions,
+                "failed": failed_executions
+            }
+        }
+    except Exception as e:
+        logger.error(f"Error retrieving metrics: {e}")
+        return {
+            "status": "error",
+            "message": "Unable to retrieve metrics"
+        }
+
+# Run the application if this script is executed directly
+if __name__ == "__main__":
+    # Configuration for running the server
+    uvicorn_config = {
+        "host": settings.HOST,
+        "port": settings.PORT,
+        "reload": settings.DEBUG,
+        "log_level": settings.LOG_LEVEL.lower()
+    }
+    
+    # Start the server
+    uvicorn.run("evoagentx.app.main:app", **uvicorn_config)

evoagentx/app/requirements.txt

@@ -0,0 +1,23 @@
+# FastAPI and ASGI server
+fastapi==0.115.10
+uvicorn==0.22.0
+pydantic==2.7.0
+pydantic-settings==2.8.1
+
+# MongoDB ODM
+motor==3.3.1
+pymongo==4.6.0
+sqlalchemy-2.0.38
+
+python-jose==3.3.0
+passlib==1.7.4
+python-multipart==0.0.6
+bcrypt==4.0.1
+celery==5.3.4
+redis==5.0.0
+pytest==7.4.2
+pytest-asyncio==0.21.0
+httpx==0.24.1
+asgi-lifespan==1.0.1
+python-dotenv==1.0.0
+loguru==0.7.3

evoagentx/app/schemas.py

@@ -0,0 +1,168 @@
+"""
+Pydantic models for request/response validation in the EvoAgentX API.
+"""
+from datetime import datetime
+from typing import Optional, List, Dict, Any # , Union
+from pydantic import BaseModel, Field # , validator
+from bson import ObjectId
+from evoagentx.app.db import AgentStatus, WorkflowStatus, ExecutionStatus
+
+# Helper for ObjectId validation
+class PyObjectId(ObjectId):
+    @classmethod
+    def __get_validators__(cls):
+        yield cls.validate
+    
+    @classmethod
+    def validate(cls, v):
+        if not ObjectId.is_valid(v):
+            raise ValueError("Invalid ObjectId")
+        return ObjectId(v)
+    
+    @classmethod
+    def __modify_schema__(cls, field_schema):
+        field_schema.update(type="string")
+
+# Base Schema Models
+class BaseSchema(BaseModel):
+    class Config:
+        allow_population_by_field_name = True
+        arbitrary_types_allowed = True
+        json_encoders = {
+            ObjectId: str,
+            datetime: lambda dt: dt.isoformat()
+        }
+
+# Agent Schemas
+class AgentCreate(BaseSchema):
+    name: str
+    description: Optional[str] = None
+    config: Dict[str, Any]
+    runtime_params: Dict[str, Any] = Field(default_factory=dict)
+    tags: List[str] = Field(default_factory=list)
+
+class AgentUpdate(BaseSchema):
+    name: Optional[str] = None
+    description: Optional[str] = None
+    config: Optional[Dict[str, Any]] = None
+    runtime_params: Optional[Dict[str, Any]] = None
+    status: Optional[AgentStatus] = None
+    tags: Optional[List[str]] = None
+
+class AgentResponse(BaseSchema):
+    id: str = Field(..., alias="_id")
+    name: str
+    description: Optional[str] = None
+    config: Dict[str, Any]
+    status: AgentStatus
+    runtime_params: Dict[str, Any]
+    created_at: datetime
+    updated_at: datetime
+    created_by: Optional[str] = None
+    tags: List[str]
+
+# Workflow Schemas
+class WorkflowStepDefinition(BaseSchema):
+    step_id: str
+    agent_id: str
+    action: str
+    input_mapping: Dict[str, str] = Field(default_factory=dict)
+    output_mapping: Dict[str, str] = Field(default_factory=dict)
+    timeout_seconds: int = 300
+    retry_count: int = 3
+    depends_on: List[str] = Field(default_factory=list)
+
+class WorkflowCreate(BaseSchema):
+    name: str
+    description: Optional[str] = None
+    definition: Dict[str, Any]
+    tags: List[str] = Field(default_factory=list)
+
+class WorkflowUpdate(BaseSchema):
+    name: Optional[str] = None
+    description: Optional[str] = None
+    definition: Optional[Dict[str, Any]] = None
+    status: Optional[WorkflowStatus] = None
+    tags: Optional[List[str]] = None
+
+class WorkflowResponse(BaseSchema):
+    id: str = Field(..., alias="_id")
+    name: str
+    description: Optional[str] = None
+    definition: Dict[str, Any]
+    agent_ids: List[str]
+    status: WorkflowStatus
+    created_at: datetime
+    updated_at: datetime
+    created_by: Optional[str] = None
+    tags: List[str]
+    version: int
+
+# Execution Schemas
+class ExecutionCreate(BaseSchema):
+    workflow_id: str
+    input_params: Dict[str, Any] = Field(default_factory=dict)
+    callback_url: Optional[str] = None
+
+class ExecutionResponse(BaseSchema):
+    id: str = Field(..., alias="_id")
+    workflow_id: str
+    status: ExecutionStatus
+    start_time: Optional[datetime] = None
+    end_time: Optional[datetime] = None
+    input_params: Dict[str, Any]
+    results: Dict[str, Any]
+    created_by: Optional[str] = None
+    step_results: Dict[str, Dict[str, Any]]
+    current_step: Optional[str] = None
+    error_message: Optional[str] = None
+    created_at: datetime
+
+class ExecutionLogResponse(BaseSchema):
+    id: str = Field(..., alias="_id")
+    workflow_id: str
+    execution_id: str
+    step_id: Optional[str] = None
+    agent_id: Optional[str] = None
+    timestamp: datetime
+    level: str
+    message: str
+    details: Dict[str, Any]
+
+# User auth schemas
+class Token(BaseSchema):
+    access_token: str
+    token_type: str
+
+class TokenPayload(BaseSchema):
+    sub: Optional[str] = None
+    exp: Optional[int] = None
+
+class UserCreate(BaseSchema):
+    email: str
+    password: str
+    full_name: Optional[str] = None
+
+class UserLogin(BaseSchema):
+    email: str
+    password: str
+
+class UserResponse(BaseSchema):
+    id: str = Field(..., alias="_id")
+    email: str
+    full_name: Optional[str] = None
+    is_active: bool
+    is_admin: bool
+    created_at: datetime
+
+# Query parameters
+class PaginationParams(BaseSchema):
+    skip: int = 0
+    limit: int = 100
+    
+class SearchParams(BaseSchema):
+    query: Optional[str] = None
+    tags: Optional[List[str]] = None
+    status: Optional[str] = None
+    start_date: Optional[datetime] = None
+    end_date: Optional[datetime] = None

evoagentx/app/security.py

@@ -0,0 +1,172 @@
+"""
+Security components for authentication and authorization.
+"""
+import jwt
+from datetime import datetime, timedelta
+from typing import Optional, Dict, Any # , List
+from passlib.context import CryptContext
+from fastapi import Depends, HTTPException, status
+from fastapi.security import OAuth2PasswordBearer
+from pydantic import BaseModel, ValidationError
+from pymongo.errors import DuplicateKeyError
+from bson import ObjectId
+
+from evoagentx.app.config import settings
+from evoagentx.app.db import Database
+from evoagentx.app.schemas import TokenPayload, UserCreate, UserResponse
+
+# Password hashing
+pwd_context = CryptContext(schemes=["bcrypt"], deprecated="auto")
+
+# OAuth2 scheme for token authentication
+oauth2_scheme = OAuth2PasswordBearer(tokenUrl=f"{settings.API_PREFIX}/auth/login")
+
+# User model for database
+class UserInDB(BaseModel):
+    _id: Optional[ObjectId] = None
+    email: str
+    hashed_password: str
+    full_name: Optional[str] = None
+    is_active: bool = True
+    is_admin: bool = False
+    created_at: datetime = datetime.utcnow()
+
+def verify_password(plain_password: str, hashed_password: str) -> bool:
+    """Verify a password against a hash."""
+    return pwd_context.verify(plain_password, hashed_password)
+
+def get_password_hash(password: str) -> str:
+    """Hash a password for storing."""
+    return pwd_context.hash(password)
+
+async def get_user_by_email(email: str) -> Optional[Dict[str, Any]]:
+    """Get a user by email."""
+    return await Database.db.users.find_one({"email": email})
+
+async def authenticate_user(email: str, password: str) -> Optional[Dict[str, Any]]:
+    """Authenticate a user by email and password."""
+    user = await get_user_by_email(email)
+    if not user:
+        return None
+    if not verify_password(password, user["hashed_password"]):
+        return None
+    if not user.get("is_active", True):
+        return None
+    return user
+
+async def create_user(user_create: UserCreate) -> UserResponse:
+    """Create a new user."""
+    # Check if user already exists
+    existing_user = await get_user_by_email(user_create.email)
+    if existing_user:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail="Email already registered"
+        )
+    
+    # Create new user
+    user_dict = user_create.dict()
+    hashed_password = get_password_hash(user_dict.pop("password"))
+    
+    new_user = {
+        "email": user_dict["email"],
+        "hashed_password": hashed_password,
+        "full_name": user_dict.get("full_name"),
+        "is_active": True,
+        "is_admin": False,
+        "created_at": datetime.utcnow()
+    }
+    
+    try:
+        insert_result = await Database.db.users.insert_one(new_user)
+        new_user["_id"] = insert_result.inserted_id
+        return UserResponse(**new_user)
+    except DuplicateKeyError:
+        raise HTTPException(
+            status_code=status.HTTP_400_BAD_REQUEST,
+            detail="Email already registered"
+        )
+
+def create_access_token(subject: str, expires_delta: Optional[timedelta] = None) -> str:
+    """Create a new JWT access token."""
+    if expires_delta:
+        expire = datetime.utcnow() + expires_delta
+    else:
+        expire = datetime.utcnow() + timedelta(minutes=settings.ACCESS_TOKEN_EXPIRE_MINUTES)
+    
+    to_encode = {"exp": expire, "sub": subject}
+    encoded_jwt = jwt.encode(to_encode, settings.SECRET_KEY, algorithm=settings.ALGORITHM)
+    return encoded_jwt
+
+async def get_current_user(token: str = Depends(oauth2_scheme)) -> Dict[str, Any]:
+    """Get the current user from a JWT token."""
+    try:
+        payload = jwt.decode(
+            token, settings.SECRET_KEY, algorithms=[settings.ALGORITHM]
+        )
+        token_data = TokenPayload(**payload)
+        
+        if datetime.fromtimestamp(token_data.exp) < datetime.utcnow():
+            raise HTTPException(
+                status_code=status.HTTP_401_UNAUTHORIZED,
+                detail="Token expired",
+                headers={"WWW-Authenticate": "Bearer"},
+            )
+    except (jwt.PyJWTError, ValidationError):
+        raise HTTPException(
+            status_code=status.HTTP_403_FORBIDDEN,
+            detail="Could not validate credentials",
+            headers={"WWW-Authenticate": "Bearer"},
+        )
+    
+    user = await get_user_by_email(token_data.sub)
+    if user is None:
+        raise HTTPException(
+            status_code=status.HTTP_404_NOT_FOUND,
+            detail="User not found"
+        )
+    
+    return user
+
+async def get_current_active_user(current_user: Dict[str, Any] = Depends(get_current_user)) -> Dict[str, Any]:
+    """Get the current active user."""
+    if not current_user.get("is_active", True):
+        raise HTTPException(status_code=400, detail="Inactive user")
+    return current_user
+
+async def get_current_admin_user(current_user: Dict[str, Any] = Depends(get_current_user)) -> Dict[str, Any]:
+    """Get the current admin user."""
+    if not current_user.get("is_admin", False):
+        raise HTTPException(
+            status_code=status.HTTP_403_FORBIDDEN,
+            detail="Not enough permissions"
+        )
+    return current_user
+
+# Initialize the users collection
+async def init_users_collection():
+    """Initialize the users collection with indexes."""
+    await Database.db.users.create_index("email", unique=True)
+    
+    # Create admin user if it doesn't exist
+    admin_email = "admin@clayx.ai"
+    admin = await get_user_by_email(admin_email)
+    if not admin:
+        admin_user = UserCreate(
+            email=admin_email,
+            password="adminpassword",  # Change this in production!
+            full_name="Admin User"
+        )
+        user_dict = admin_user.dict()
+        hashed_password = get_password_hash(user_dict["password"])
+        
+        new_admin = {
+            "email": admin_email,
+            "hashed_password": hashed_password,
+            "full_name": "Admin User",
+            "is_active": True,
+            "is_admin": True,
+            "created_at": datetime.utcnow()
+        }
+        
+        await Database.db.users.insert_one(new_admin)

evoagentx/app/services.py

@@ -0,0 +1,463 @@
+"""
+Business logic for agents, workflows, and executions.
+"""
+import logging
+# import asyncio
+from datetime import datetime
+from typing import List, Dict, Any, Optional, Tuple
+from bson import ObjectId
+
+from evoagentx.app.db import (
+    Database, # Agent, Workflow, WorkflowExecution, ExecutionLog,
+    AgentStatus, WorkflowStatus, ExecutionStatus
+)
+from evoagentx.app.schemas import (
+    AgentCreate, AgentUpdate, WorkflowCreate, WorkflowUpdate, 
+    ExecutionCreate, PaginationParams, SearchParams
+)
+
+logger = logging.getLogger(__name__)
+
+# Agent Service
+class AgentService:
+    @staticmethod
+    async def create_agent(agent_data: AgentCreate, user_id: Optional[str] = None) -> Dict[str, Any]:
+        """Create a new agent."""
+        agent_dict = agent_data.dict()
+        agent_dict["created_by"] = user_id
+        agent_dict["created_at"] = datetime.utcnow()
+        agent_dict["updated_at"] = agent_dict["created_at"]
+        agent_dict["status"] = AgentStatus.CREATED
+        
+        # Validate agent exists with the same name
+        existing_agent = await Database.agents.find_one({"name": agent_dict["name"]})
+        if existing_agent:
+            raise ValueError(f"Agent with name '{agent_dict['name']}' already exists")
+        
+        result = await Database.agents.insert_one(agent_dict)
+        agent_dict["_id"] = result.inserted_id
+        
+        logger.info(f"Created agent {agent_dict['name']} with ID {result.inserted_id}")
+        
+        return agent_dict
+    
+    @staticmethod
+    async def get_agent(agent_id: str) -> Optional[Dict[str, Any]]:
+        """Get an agent by ID."""
+        if not ObjectId.is_valid(agent_id):
+            raise ValueError(f"Invalid agent ID: {agent_id}")
+            
+        agent = await Database.agents.find_one({"_id": ObjectId(agent_id)})
+        return agent
+    
+    @staticmethod
+    async def get_agent_by_name(name: str) -> Optional[Dict[str, Any]]:
+        """Get an agent by name."""
+        return await Database.agents.find_one({"name": name})
+    
+    @staticmethod
+    async def update_agent(agent_id: str, agent_data: AgentUpdate) -> Optional[Dict[str, Any]]:
+        """Update an agent."""
+        if not ObjectId.is_valid(agent_id):
+            raise ValueError(f"Invalid agent ID: {agent_id}")
+            
+        agent = await Database.agents.find_one({"_id": ObjectId(agent_id)})
+        if not agent:
+            return None
+        
+        update_data = agent_data.dict(exclude_unset=True)
+        update_data["updated_at"] = datetime.utcnow()
+        
+        if "name" in update_data:
+            # Check if the new name already exists
+            existing = await Database.agents.find_one({
+                "name": update_data["name"],
+                "_id": {"$ne": ObjectId(agent_id)}
+            })
+            if existing:
+                raise ValueError(f"Agent with name '{update_data['name']}' already exists")
+        
+        await Database.agents.update_one(
+            {"_id": ObjectId(agent_id)},
+            {"$set": update_data}
+        )
+        
+        updated_agent = await Database.agents.find_one({"_id": ObjectId(agent_id)})
+        logger.info(f"Updated agent {agent_id}")
+        
+        return updated_agent
+    
+    @staticmethod
+    async def delete_agent(agent_id: str) -> bool:
+        """Delete an agent."""
+        if not ObjectId.is_valid(agent_id):
+            raise ValueError(f"Invalid agent ID: {agent_id}")
+            
+        # Check if agent is used in any workflows
+        workflow_count = await Database.workflows.count_documents({"agent_ids": agent_id})
+        if workflow_count > 0:
+            raise ValueError(f"Cannot delete agent {agent_id} as it is used in {workflow_count} workflows")
+        
+        result = await Database.agents.delete_one({"_id": ObjectId(agent_id)})
+        if result.deleted_count:
+            logger.info(f"Deleted agent {agent_id}")
+            return True
+        return False
+    
+    @staticmethod
+    async def list_agents(
+        params: PaginationParams, 
+        search: Optional[SearchParams] = None
+    ) -> Tuple[List[Dict[str, Any]], int]:
+        """List agents with pagination and search."""
+        query = {}
+        
+        if search:
+            if search.query:
+                query["$text"] = {"$search": search.query}
+            
+            if search.tags:
+                query["tags"] = {"$all": search.tags}
+            
+            if search.status:
+                query["status"] = search.status
+            
+            if search.start_date and search.end_date:
+                query["created_at"] = {
+                    "$gte": search.start_date,
+                    "$lte": search.end_date
+                }
+            elif search.start_date:
+                query["created_at"] = {"$gte": search.start_date}
+            elif search.end_date:
+                query["created_at"] = {"$lte": search.end_date}
+        
+        total = await Database.agents.count_documents(query)
+        
+        cursor = Database.agents.find(query)\
+            .sort("created_at", -1)\
+            .skip(params.skip)\
+            .limit(params.limit)
+        
+        agents = await cursor.to_list(length=params.limit)
+        return agents, total
+
+# Workflow Service
+class WorkflowService:
+    @staticmethod
+    async def create_workflow(workflow_data: WorkflowCreate, user_id: Optional[str] = None) -> Dict[str, Any]:
+        """Create a new workflow."""
+        workflow_dict = workflow_data.dict()
+        workflow_dict["created_by"] = user_id
+        workflow_dict["created_at"] = datetime.utcnow()
+        workflow_dict["updated_at"] = workflow_dict["created_at"]
+        workflow_dict["status"] = WorkflowStatus.CREATED
+        workflow_dict["version"] = 1
+        
+        # Extract agent IDs from the workflow definition
+        agent_ids = set()
+        
+        # Extract agent IDs from steps
+        steps = workflow_dict["definition"].get("steps", [])
+        for step in steps:
+            if "agent_id" in step:
+                agent_id = step["agent_id"]
+                # Validate agent exists
+                agent = await AgentService.get_agent(agent_id)
+                if not agent:
+                    raise ValueError(f"Agent with ID {agent_id} does not exist")
+                agent_ids.add(agent_id)
+        
+        workflow_dict["agent_ids"] = list(agent_ids)
+        
+        # Check for existing workflow with the same name
+        existing = await Database.workflows.find_one({"name": workflow_dict["name"]})
+        if existing:
+            raise ValueError(f"Workflow with name '{workflow_dict['name']}' already exists")
+        
+        result = await Database.workflows.insert_one(workflow_dict)
+        workflow_dict["_id"] = result.inserted_id
+        
+        logger.info(f"Created workflow {workflow_dict['name']} with ID {result.inserted_id}")
+        
+        return workflow_dict
+    
+    @staticmethod
+    async def get_workflow(workflow_id: str) -> Optional[Dict[str, Any]]:
+        """Get a workflow by ID."""
+        if not ObjectId.is_valid(workflow_id):
+            raise ValueError(f"Invalid workflow ID: {workflow_id}")
+        workflow = await Database.workflows.find_one({"_id": ObjectId(workflow_id)})
+        return workflow
+    
+    @staticmethod
+    async def get_workflow_by_name(name: str) -> Optional[Dict[str, Any]]:
+        """Get a workflow by name."""
+        return await Database.workflows.find_one({"name": name})
+    
+    @staticmethod
+    async def update_workflow(workflow_id: str, workflow_data: WorkflowUpdate) -> Optional[Dict[str, Any]]:
+        """Update a workflow."""
+        if not ObjectId.is_valid(workflow_id):
+            raise ValueError(f"Invalid workflow ID: {workflow_id}")
+            
+        workflow = await Database.workflows.find_one({"_id": ObjectId(workflow_id)})
+        if not workflow:
+            return None
+        
+        update_data = workflow_data.dict(exclude_unset=True)
+        update_data["updated_at"] = datetime.utcnow()
+        
+        # Update version if definition changes
+        if "definition" in update_data:
+            update_data["version"] = workflow.get("version", 1) + 1
+            
+            # Extract agent IDs from the updated workflow definition
+            agent_ids = set()
+            steps = update_data["definition"].get("steps", [])
+            for step in steps:
+                if "agent_id" in step:
+                    agent_id = step["agent_id"]
+                    # Validate agent exists
+                    agent = await AgentService.get_agent(agent_id)
+                    if not agent:
+                        raise ValueError(f"Agent with ID {agent_id} does not exist")
+                    agent_ids.add(agent_id)
+            
+            update_data["agent_ids"] = list(agent_ids)
+        
+        # Check for name conflict if name is being updated
+        if "name" in update_data:
+            existing = await Database.workflows.find_one({
+                "name": update_data["name"],
+                "_id": {"$ne": ObjectId(workflow_id)}
+            })
+            if existing:
+                raise ValueError(f"Workflow with name '{update_data['name']}' already exists")
+        
+        await Database.workflows.update_one(
+            {"_id": ObjectId(workflow_id)},
+            {"$set": update_data}
+        )
+        
+        updated_workflow = await Database.workflows.find_one({"_id": ObjectId(workflow_id)})
+        logger.info(f"Updated workflow {workflow_id}")
+        
+        return updated_workflow
+    
+    @staticmethod
+    async def delete_workflow(workflow_id: str) -> bool:
+        """Delete a workflow."""
+        if not ObjectId.is_valid(workflow_id):
+            raise ValueError(f"Invalid workflow ID: {workflow_id}")
+        
+        # Check if workflow has any ongoing or recent executions
+        recent_executions = await Database.executions.count_documents({
+            "workflow_id": workflow_id,
+            "status": {"$in": [
+                ExecutionStatus.PENDING, 
+                ExecutionStatus.RUNNING
+            ]}
+        })
+        
+        if recent_executions > 0:
+            raise ValueError(f"Cannot delete workflow {workflow_id} with {recent_executions} active executions")
+
+        result = await Database.workflows.delete_one({"_id": ObjectId(workflow_id)})
+        if result.deleted_count:
+            # Delete associated execution logs
+            await Database.logs.delete_many({"workflow_id": workflow_id})
+            await Database.executions.delete_many({"workflow_id": workflow_id})
+            
+            logger.info(f"Deleted workflow {workflow_id}")
+            return True
+        return False
+    
+    @staticmethod
+    async def list_workflows(
+        params: PaginationParams, 
+        search: Optional[SearchParams] = None
+    ) -> Tuple[List[Dict[str, Any]], int]:
+        """List workflows with pagination and search."""
+        query = {}
+        
+        if search:
+            if search.query:
+                query["$text"] = {"$search": search.query}
+            
+            if search.tags:
+                query["tags"] = {"$all": search.tags}
+            
+            if search.status:
+                query["status"] = search.status
+            
+            if search.start_date and search.end_date:
+                query["created_at"] = {
+                    "$gte": search.start_date,
+                    "$lte": search.end_date
+                }
+            elif search.start_date:
+                query["created_at"] = {"$gte": search.start_date}
+            elif search.end_date:
+                query["created_at"] = {"$lte": search.end_date}
+        
+        total = await Database.workflows.count_documents(query)
+        
+        cursor = Database.workflows.find(query)\
+            .sort("created_at", -1)\
+            .skip(params.skip)\
+            .limit(params.limit)
+        
+        workflows = await cursor.to_list(length=params.limit)
+        return workflows, total
+
+# Workflow Execution Service
+class WorkflowExecutionService:
+    @staticmethod
+    async def create_execution(execution_data: ExecutionCreate, user_id: Optional[str] = None) -> Dict[str, Any]:
+        """Create a new workflow execution."""
+        # Validate workflow exists
+        workflow = await WorkflowService.get_workflow(execution_data.workflow_id)
+        if not workflow:
+            raise ValueError(f"Workflow {execution_data.workflow_id} not found")
+        
+        # Prepare execution document
+        execution_dict = {
+            "workflow_id": execution_data.workflow_id,
+            "status": ExecutionStatus.PENDING,
+            "start_time": datetime.utcnow(),
+            "input_params": execution_data.input_params,
+            "created_by": user_id,
+            "created_at": datetime.utcnow(),
+            "step_results": {},
+            "current_step": None,
+            "results": {},
+            "error_message": None
+        }
+        
+        # Insert execution record
+        result = await Database.executions.insert_one(execution_dict)
+        execution_dict["_id"] = result.inserted_id
+        
+        logger.info(f"Created workflow execution {result.inserted_id}")
+        
+        # Optional: Queue execution for async processing
+        # This would typically use a task queue like Celery
+        # await execute_workflow_async.delay(execution_dict)
+        
+        return execution_dict
+    
+    @staticmethod
+    async def get_execution(execution_id: str) -> Optional[Dict[str, Any]]:
+        """Get a workflow execution by ID."""
+        if not ObjectId.is_valid(execution_id):
+            raise ValueError(f"Invalid execution ID: {execution_id}")
+            
+        execution = await Database.executions.find_one({"_id": ObjectId(execution_id)})
+        return execution
+    
+    @staticmethod
+    async def update_execution_status(execution_id: str, status: ExecutionStatus, error_message: Optional[str] = None) -> Optional[Dict[str, Any]]:
+        """Update execution status."""
+        if not ObjectId.is_valid(execution_id):
+            raise ValueError(f"Invalid execution ID: {execution_id}")
+        
+        update_data = {
+            "status": status,
+            "updated_at": datetime.utcnow()
+        }
+        
+        if status in [ExecutionStatus.COMPLETED, ExecutionStatus.FAILED, ExecutionStatus.CANCELLED]:
+            update_data["end_time"] = datetime.utcnow()
+        
+        if error_message:
+            update_data["error_message"] = error_message
+        
+        result = await Database.executions.find_one_and_update(
+            {"_id": ObjectId(execution_id)},
+            {"$set": update_data},
+            return_document=True
+        )
+        
+        return result
+    
+    @staticmethod
+    async def list_executions(
+        workflow_id: Optional[str] = None,
+        params: PaginationParams = PaginationParams(), 
+        search: Optional[SearchParams] = None
+    ) -> Tuple[List[Dict[str, Any]], int]:
+        """List workflow executions with pagination and search."""
+        query = {}
+        
+        if workflow_id:
+            query["workflow_id"] = workflow_id
+        
+        if search:
+            if search.status:
+                query["status"] = search.status
+            
+            if search.start_date and search.end_date:
+                query["created_at"] = {
+                    "$gte": search.start_date,
+                    "$lte": search.end_date
+                }
+            elif search.start_date:
+                query["created_at"] = {"$gte": search.start_date}
+            elif search.end_date:
+                query["created_at"] = {"$lte": search.end_date}
+        
+        total = await Database.executions.count_documents(query)
+        
+        cursor = Database.executions.find(query)\
+            .sort("created_at", -1)\
+            .skip(params.skip)\
+            .limit(params.limit)
+        
+        executions = await cursor.to_list(length=params.limit)
+        return executions, total
+    
+    @staticmethod
+    async def log_execution_event(
+        workflow_id: str, 
+        execution_id: str, 
+        message: str,
+        step_id: Optional[str] = None, 
+        agent_id: Optional[str] = None, 
+        level: str = "INFO", 
+        details: Optional[Dict[str, Any]] = None
+    ) -> Dict[str, Any]:
+        """Log an event in a workflow execution."""
+        log_entry = {
+            "workflow_id": workflow_id,
+            "execution_id": execution_id,
+            "step_id": step_id,
+            "agent_id": agent_id,
+            "timestamp": datetime.utcnow(),
+            "level": level,
+            "message": message,
+            "details": details or {}
+        }
+        
+        result = await Database.logs.insert_one(log_entry)
+        log_entry["_id"] = result.inserted_id
+        
+        return log_entry
+    
+    @staticmethod
+    async def get_execution_logs(
+        execution_id: str, 
+        params: PaginationParams = PaginationParams()
+    ) -> Tuple[List[Dict[str, Any]], int]:
+        """Retrieve logs for a specific execution."""
+        query = {"execution_id": execution_id}
+        
+        total = await Database.logs.count_documents(query)
+        
+        cursor = Database.logs.find(query)\
+            .sort("timestamp", 1)\
+            .skip(params.skip)\
+            .limit(params.limit)
+        
+        logs = await cursor.to_list(length=params.limit)
+        return logs, total

evoagentx/benchmark/.ipynb_checkpoints/Untitled-checkpoint.ipynb

@@ -0,0 +1,6 @@
+{
+ "cells": [],
+ "metadata": {},
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

evoagentx/benchmark/.ipynb_checkpoints/test_load_json-checkpoint.ipynb

@@ -0,0 +1,570 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "385cefbd",
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "ImportError",
+     "evalue": "attempted relative import with no known parent package",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mImportError\u001b[0m                               Traceback (most recent call last)",
+      "Cell \u001b[0;32mIn[1], line 7\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mshutil\u001b[39;00m\n\u001b[1;32m      5\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mtyping\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m Union, Any, Callable, List, Dict, Tuple\n\u001b[0;32m----> 7\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mscicode\u001b[39;00m\n\u001b[1;32m      9\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mnumpy\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01mnp\u001b[39;00m  \u001b[38;5;66;03m# Many SciCode tests use numpy\u001b[39;00m\n\u001b[1;32m     10\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mbenchmark\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m CodingBenchmark\n",
+      "File \u001b[0;32m/gpfs/radev/pi/ying_rex/tl688/selfevolve/EvoAgentX/evoagentx/benchmark/scicode.py:10\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mscicode\u001b[39;00m\n\u001b[1;32m      9\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mnumpy\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01mnp\u001b[39;00m  \u001b[38;5;66;03m# Many SciCode tests use numpy\u001b[39;00m\n\u001b[0;32m---> 10\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mbenchmark\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m CodingBenchmark\n\u001b[1;32m     11\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mcore\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mlogging\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m logger\n\u001b[1;32m     12\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mutils\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mutils\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m download_file\n",
+      "\u001b[0;31mImportError\u001b[0m: attempted relative import with no known parent package"
+     ]
+    }
+   ],
+   "source": [
+    "import os\n",
+    "import re\n",
+    "import gzip\n",
+    "import shutil\n",
+    "from typing import Union, Any, Callable, List, Dict, Tuple\n",
+    "\n",
+    "import scicode\n",
+    "\n",
+    "import numpy as np  # Many SciCode tests use numpy\n",
+    "from .benchmark import CodingBenchmark\n",
+    "from ..core.logging import logger\n",
+    "from ..utils.utils import download_file\n",
+    "from ..core.module_utils import load_json\n",
+    "from ..utils.aflow_utils.data_utils import AFLOW_DATASET_FILES_MAP, download_aflow_benchmark_data\n",
+    "\n",
+    "\n",
+    "# ----------------------------\n",
+    "# Raw SciCode (community) data\n",
+    "# ----------------------------\n",
+    "\n",
+    "SCICODE_DEFAULT_URL = \"https://raw.githubusercontent.com/scicode-bench/scicode/main/data/scicode.jsonl.gz\"  # If you mirror elsewhere, update here.\n",
+    "\n",
+    "\n",
+    "def download_raw_scicode_data(save_folder: str, url: str = SCICODE_DEFAULT_URL) -> str:\n",
+    "    \"\"\"\n",
+    "    Download and unzip the raw SciCode jsonl(.gz) to `save_folder`.\n",
+    "\n",
+    "    Returns:\n",
+    "        str: Path to the unzipped jsonl file.\n",
+    "    \"\"\"\n",
+    "    os.makedirs(save_folder, exist_ok=True)\n",
+    "    gz_path = os.path.join(save_folder, \"scicode.jsonl.gz\")\n",
+    "    jsonl_path = os.path.join(save_folder, \"scicode.jsonl\")\n",
+    "\n",
+    "    logger.info(f\"Downloading SciCode data from {url} ...\")\n",
+    "    download_file(url=url, save_file=gz_path)\n",
+    "\n",
+    "    logger.info(\"Unzipping SciCode data ...\")\n",
+    "    with gzip.open(gz_path, \"rb\") as f_in, open(jsonl_path, \"wb\") as f_out:\n",
+    "        shutil.copyfileobj(f_in, f_out)\n",
+    "    if os.path.exists(gz_path):\n",
+    "        os.remove(gz_path)\n",
+    "\n",
+    "    return jsonl_path\n",
+    "\n",
+    "\n",
+    "# ----------------------------\n",
+    "# Schema helpers\n",
+    "# ----------------------------\n",
+    "\n",
+    "def _extract_entry_point_from_header(header: str) -> str:\n",
+    "    \"\"\"\n",
+    "    Given a SciCode 'function_header' string like:\n",
+    "        \"def get_alpha(recvec, alpha_scaling=5):\\n    '''...'''\"\n",
+    "    return \"get_alpha\".\n",
+    "    \"\"\"\n",
+    "    m = re.search(r\"def\\s+([A-Za-z_][A-Za-z0-9_]*)\\s*\\(\", header)\n",
+    "    if not m:\n",
+    "        raise ValueError(\"Could not parse entry point from function_header\")\n",
+    "    return m.group(1)\n",
+    "\n",
+    "\n",
+    "def _coerce_scicode_row_to_examples(row: Dict[str, Any]) -> List[Dict[str, Any]]:\n",
+    "    \"\"\"\n",
+    "    SciCode rows may contain a single task or multiple step tasks.\n",
+    "    We normalize them to a list of examples with a unified structure:\n",
+    "        {\n",
+    "            \"task_id\": \"SciCode/<name>#<sub_id>\",\n",
+    "            \"prompt\": <function_header + optional docstring block>,\n",
+    "            \"entry_point\": <func_name>,\n",
+    "            \"canonical_solution\": <ground_truth_code>,\n",
+    "            \"tests\": List[str],  # list of python test snippets\n",
+    "            \"imports\": str       # optional import prelude (e.g., 'import numpy as np')\n",
+    "        }\n",
+    "    \"\"\"\n",
+    "    examples: List[Dict[str, Any]] = []\n",
+    "\n",
+    "    name = str(row[0]) if 0 in row or isinstance(row, list) else str(row.get(\"name\", \"unknown\"))\n",
+    "    # Different dumps can be list-based or dict-based; support both:\n",
+    "    if isinstance(row, list):\n",
+    "        # Heuristic index layout (based on the example provided by the user):\n",
+    "        # [name, <maybe_int>, description, <maybe empty>, docstring, imports, steps(list[dict]) or code, tests(list[str]) or None]\n",
+    "        # We will try to find keys by semantic type\n",
+    "        description = None\n",
+    "        doc_or_header = None\n",
+    "        imports_block = None\n",
+    "        steps_or_code = None\n",
+    "        tests = None\n",
+    "\n",
+    "        # Try assigning by scanning\n",
+    "        for item in row:\n",
+    "            if isinstance(item, str) and item.strip().startswith('\"\"\"'):\n",
+    "                # docstring/prompt block for the top-level task\n",
+    "                doc_or_header = item\n",
+    "            elif isinstance(item, str) and (item.startswith(\"import \") or \"from \" in item):\n",
+    "                imports_block = item\n",
+    "            elif isinstance(item, list):\n",
+    "                # Could be steps OR tests\n",
+    "                if item and isinstance(item[0], dict) and \"function_header\" in item[0]:\n",
+    "                    steps_or_code = item\n",
+    "                elif item and isinstance(item[0], str) and item[0].strip().startswith((\"ref\", \"assert\", \"from \")):\n",
+    "                    tests = item\n",
+    "            elif isinstance(item, dict):\n",
+    "                # Some SciCode variants may directly be dicts per step; treat as steps\n",
+    "                steps_or_code = [item]\n",
+    "\n",
+    "        # If we have step dictionaries, produce one example per step\n",
+    "        if isinstance(steps_or_code, list) and steps_or_code and isinstance(steps_or_code[0], dict):\n",
+    "            for idx, step in enumerate(steps_or_code):\n",
+    "                header = step.get(\"function_header\") or step.get(\"header\") or \"\"\n",
+    "                code = step.get(\"ground_truth_code\") or step.get(\"solution\") or \"\"\n",
+    "                step_tests = step.get(\"test_cases\") or []\n",
+    "                entry_point = _extract_entry_point_from_header(header)\n",
+    "                prompt = header  # keep header as the model prompt (header + docstring already embedded)\n",
+    "                examples.append(\n",
+    "                    {\n",
+    "                        \"task_id\": f\"SciCode/{name}#step{idx+1}\",\n",
+    "                        \"prompt\": prompt,\n",
+    "                        \"entry_point\": entry_point,\n",
+    "                        \"canonical_solution\": code,\n",
+    "                        \"tests\": step_tests,\n",
+    "                        \"imports\": imports_block or \"\",\n",
+    "                    }\n",
+    "                )\n",
+    "        else:\n",
+    "            # Single task variant: expect a combined \"function_header\" + \"ground_truth_code\" + \"test_cases\" in the row\n",
+    "            # Try to detect them from the large code string block if present.\n",
+    "            # Fall back to no-op if missing.\n",
+    "            # NOTE: The user’s example shows a consolidated block near the end; we’ll try to parse it.\n",
+    "            code_blob = None\n",
+    "            for item in row:\n",
+    "                if isinstance(item, str) and \"def \" in item and \"return\" in item:\n",
+    "                    code_blob = item\n",
+    "                    break\n",
+    "            # Try to split the big blob into multiple functions; evaluate the last one as the main if we cannot find header separately.\n",
+    "            if code_blob:\n",
+    "                # Heuristic: the last \"def ...\" in the blob is the target entry point\n",
+    "                headers = list(re.finditer(r\"(?ms)^(def\\s+[A-Za-z_][A-Za-z0-9_]*\\s*\\(.*?\\):\\s*\\n)\", code_blob))\n",
+    "                if headers:\n",
+    "                    last_header = headers[-1].group(1)\n",
+    "                    entry_point = _extract_entry_point_from_header(last_header)\n",
+    "                else:\n",
+    "                    entry_point = \"solution\"\n",
+    "\n",
+    "                # We will treat entire blob as canonical_solution and create a minimal prompt from the docstring if present\n",
+    "                prompt = doc_or_header or f\"def {entry_point}(*args, **kwargs):\\n    '''Fill in the function body.'''\\n    ...\"\n",
+    "                examples.append(\n",
+    "                    {\n",
+    "                        \"task_id\": f\"SciCode/{name}\",\n",
+    "                        \"prompt\": prompt,\n",
+    "                        \"entry_point\": entry_point,\n",
+    "                        \"canonical_solution\": code_blob,\n",
+    "                        \"tests\": tests or [],\n",
+    "                        \"imports\": imports_block or \"\",\n",
+    "                    }\n",
+    "                )\n",
+    "\n",
+    "    else:\n",
+    "        # Dict-style row (fallback): expect keys by name\n",
+    "        steps = row.get(\"steps\", [])\n",
+    "        imports_block = row.get(\"imports\", \"\")\n",
+    "        task_name = row.get(\"name\", \"unknown\")\n",
+    "\n",
+    "        if steps:\n",
+    "            for idx, step in enumerate(steps):\n",
+    "                header = step.get(\"function_header\", \"\")\n",
+    "                code = step.get(\"ground_truth_code\", \"\")\n",
+    "                step_tests = step.get(\"test_cases\", [])\n",
+    "                entry_point = _extract_entry_point_from_header(header)\n",
+    "                examples.append(\n",
+    "                    {\n",
+    "                        \"task_id\": f\"SciCode/{task_name}#step{idx+1}\",\n",
+    "                        \"prompt\": header,\n",
+    "                        \"entry_point\": entry_point,\n",
+    "                        \"canonical_solution\": code,\n",
+    "                        \"tests\": step_tests,\n",
+    "                        \"imports\": imports_block or \"\",\n",
+    "                    }\n",
+    "                )\n",
+    "        else:\n",
+    "            header = row.get(\"function_header\", \"\")\n",
+    "            code = row.get(\"ground_truth_code\", \"\")\n",
+    "            tests = row.get(\"test_cases\", [])\n",
+    "            entry_point = _extract_entry_point_from_header(header) if header else \"solution\"\n",
+    "            prompt = header or f\"def {entry_point}(*args, **kwargs):\\n    pass\"\n",
+    "            examples.append(\n",
+    "                {\n",
+    "                    \"task_id\": f\"SciCode/{task_name}\",\n",
+    "                    \"prompt\": prompt,\n",
+    "                    \"entry_point\": entry_point,\n",
+    "                    \"canonical_solution\": code,\n",
+    "                    \"tests\": tests,\n",
+    "                    \"imports\": imports_block or \"\",\n",
+    "                }\n",
+    "            )\n",
+    "\n",
+    "    return examples\n",
+    "\n",
+    "\n",
+    "def load_scicode_data(jsonl_path: str) -> List[Dict[str, Any]]:\n",
+    "    \"\"\"\n",
+    "    Load SciCode jsonl and expand into normalized examples.\n",
+    "    \"\"\"\n",
+    "    raw = load_json(jsonl_path, type=\"jsonl\")\n",
+    "    all_examples: List[Dict[str, Any]] = []\n",
+    "    for row in raw:\n",
+    "        try:\n",
+    "            all_examples.extend(_coerce_scicode_row_to_examples(row))\n",
+    "        except Exception as e:\n",
+    "            logger.warning(f\"[SciCode] Skipping a malformed row due to: {e}\")\n",
+    "    return all_examples\n",
+    "\n",
+    "\n",
+    "# ----------------------------\n",
+    "# Benchmark classes\n",
+    "# ----------------------------\n",
+    "\n",
+    "class SciCode(CodingBenchmark):\n",
+    "    \"\"\"\n",
+    "    Benchmark class for evaluating code generation on SciCode.\n",
+    "\n",
+    "    SciCode problems provide:\n",
+    "      - function_header (prompt stub)\n",
+    "      - ground_truth_code (reference implementation)\n",
+    "      - test_cases (list[str] of python asserts)\n",
+    "\n",
+    "    We normalize each item and evaluate by executing the candidate implementation\n",
+    "    against the provided test cases. Since many SciCode tests reference a variable\n",
+    "    named `target`, we heuristically pre-compute `target` from the reference\n",
+    "    implementation when necessary, or set it to True for boolean-allclose tests.\n",
+    "    \"\"\"\n",
+    "\n",
+    "    def __init__(self, path: str = None, mode: str = \"all\", timeout: int = 60, k: Union[int, list] = 1, **kwargs):\n",
+    "        path = os.path.expanduser(path or \"~/.evoagentx/data/scicode\")\n",
+    "        self.k = k\n",
+    "        super().__init__(name=type(self).__name__, path=path, mode=mode, timeout=timeout, **kwargs)\n",
+    "\n",
+    "    # ---------- Data loading ----------\n",
+    "\n",
+    "    def _load_data(self):\n",
+    "        data_path = os.path.join(self.path, \"scicode.jsonl\")\n",
+    "        if not os.path.exists(data_path):\n",
+    "            data_path = download_raw_scicode_data(self.path)\n",
+    "\n",
+    "        # For SciCode, we place everything into \"test\" split by default.\n",
+    "\n",
+    "        if self.mode in (\"dev\", \"all\"):\n",
+    "            self._dev_data = load_scicode_data(\"/home/tl688/pitl688/selfevolve/SciCode/eval/data/subproblems_dev.jsonl\")\n",
+    "        if self.mode in (\"test\", \"all\"):\n",
+    "            self._test_data = load_scicode_data(\"/home/tl688/pitl688/selfevolve/SciCode/eval/data/subproblems_test.jsonl\")\n",
+    "\n",
+    "    def _get_label(self, example: Any):\n",
+    "        \"\"\"\n",
+    "        For SciCode we treat the label as the full test suite plus metadata.\n",
+    "        \"\"\"\n",
+    "        return {\n",
+    "            \"task_id\": example[\"task_id\"],\n",
+    "            \"entry_point\": example[\"entry_point\"],\n",
+    "            \"tests\": example.get(\"tests\", []),\n",
+    "            \"canonical_solution\": example.get(\"canonical_solution\", \"\"),\n",
+    "            \"imports\": example.get(\"imports\", \"\"),\n",
+    "        }\n",
+    "\n",
+    "    def _get_id(self, example: Any):\n",
+    "        return example[\"task_id\"]\n",
+    "\n",
+    "    # ---------- Evaluation ----------\n",
+    "\n",
+    "    @staticmethod\n",
+    "    def _build_reference_namespace(imports: str, canonical_solution: str) -> Dict[str, Any]:\n",
+    "        \"\"\"\n",
+    "        Build an execution namespace that defines the reference function.\n",
+    "        \"\"\"\n",
+    "        ns: Dict[str, Any] = {\"np\": np, \"scicode\":scicode}\n",
+    "        if imports:\n",
+    "            exec(imports, ns, ns)  # e.g., \"import numpy as np\\nfrom scipy.special import erfc\"\n",
+    "        if canonical_solution:\n",
+    "            exec(canonical_solution, ns, ns)\n",
+    "        return ns\n",
+    "\n",
+    "    @staticmethod\n",
+    "    def _extract_candidate_exprs_from_test(test_src: str) -> List[str]:\n",
+    "        \"\"\"\n",
+    "        Heuristically extract expressions that are compared against `target` inside np.allclose(..., target)\n",
+    "        or equality checks like \"== target\" / \", target)\" etc. Returns a list of python expressions (as strings)\n",
+    "        that we should evaluate with the *reference* implementation to generate `target`.\n",
+    "\n",
+    "        This is a pragmatic parser covering the most common SciCode patterns.\n",
+    "        \"\"\"\n",
+    "        exprs: List[str] = []\n",
+    "\n",
+    "        # Pattern A: np.allclose( <expr>, target )\n",
+    "        for m in re.finditer(r\"np\\.allclose\\s*\\(\\s*(?P<expr>.+?)\\s*,\\s*target\\s*\\)\", test_src, flags=re.DOTALL):\n",
+    "            exprs.append(m.group(\"expr\"))\n",
+    "\n",
+    "        # Pattern B: assert <expr> == target\n",
+    "        for m in re.finditer(r\"assert\\s+(?P<expr>.+?)\\s*==\\s*target\", test_src):\n",
+    "            exprs.append(m.group(\"expr\"))\n",
+    "\n",
+    "        # Pattern C: assert <expr>, target  (when the first arg should be True)\n",
+    "        # In this case, target is expected to be True; no need to compute it.\n",
+    "        # We'll handle by leaving exprs empty and later default target=True.\n",
+    "\n",
+    "        # Pattern D: Using slices like target[0], target[1] — we try to recover by\n",
+    "        # extracting both left-hand expressions in the same line in order:\n",
+    "        #   np.allclose(func(...)[0], target[0]) and np.allclose(func(...)[1], target[1])\n",
+    "        # Already captured by Pattern A; expr may include \"[0]\" or \"[1]\".\n",
+    "        return exprs\n",
+    "\n",
+    "    @staticmethod\n",
+    "    def _compute_target_list(exprs: List[str], ref_ns: Dict[str, Any]) -> Any:\n",
+    "        \"\"\"\n",
+    "        Given a list of expressions (strings), evaluate them in the reference namespace.\n",
+    "        If multiple expressions are found, we pack them into a tuple in the same order.\n",
+    "        If no expression found, return True (to support tests of the form `assert <bool>, target`).\n",
+    "        \"\"\"\n",
+    "        if not exprs:\n",
+    "            return True\n",
+    "        values = []\n",
+    "        for ex in exprs:\n",
+    "            # Safety: limit builtins\n",
+    "            local_ns: Dict[str, Any] = {}\n",
+    "            val = eval(ex, ref_ns, local_ns)\n",
+    "            values.append(val)\n",
+    "        if len(values) == 1:\n",
+    "            return values[0]\n",
+    "        return tuple(values)\n",
+    "\n",
+    "    def _make_harness(self, task_id: str, entry_point: str, imports: str, canonical_solution: str, tests: List[str], candidate_src: str) -> str:\n",
+    "        \"\"\"\n",
+    "        Construct an executable harness that:\n",
+    "          1) Defines imports\n",
+    "          2) Defines candidate implementation (prompt + candidate completion)\n",
+    "          3) Pre-computes `target` using the reference implementation for each test (heuristics)\n",
+    "          4) Executes the original test snippet with `target` bound.\n",
+    "        We run each test independently within the same process, stopping on first failure.\n",
+    "        \"\"\"\n",
+    "        # We'll build a block that iterates tests in Python.\n",
+    "        # We cannot dynamically pass `target` into a raw `assert` snippet without executing it;\n",
+    "        # so for each test, we will:\n",
+    "        #   a) compute target in a separate namespace using reference function,\n",
+    "        #   b) then execute the original test with the candidate function and that target.\n",
+    "        # This is orchestrated by the benchmark runtime (not inside the user env).\n",
+    "\n",
+    "        # NOTE: actual orchestration happens in `evaluate()` by repeated calls to `check_solution`;\n",
+    "        # here we only prepare the body (candidate code). The unit tests are executed by the\n",
+    "        # framework’s sand-boxed executor using `test` passed in.\n",
+    "\n",
+    "        # We keep the candidate_src as-is. The imports are prepended at runtime via the test body.\n",
+    "        return candidate_src\n",
+    "\n",
+    "    def handle_special_cases(self, task_id: str, solution: str, test: str) -> Tuple[str, str]:\n",
+    "        \"\"\"\n",
+    "        Hook: adjust solution/test for edge cases in SciCode, if needed.\n",
+    "        Currently, we leave as-is and fallback to the base handler.\n",
+    "        \"\"\"\n",
+    "        return super().handle_special_cases(task_id=task_id, solution=solution, test=test)\n",
+    "\n",
+    "    def evaluate(self, prediction: Any, label: Any) -> dict:\n",
+    "        \"\"\"\n",
+    "        Evaluate candidate solution(s) against SciCode test cases.\n",
+    "\n",
+    "        Strategy:\n",
+    "          - For each candidate solution:\n",
+    "              - For each test snippet:\n",
+    "                  1) Build reference namespace; compute `target` (heuristics).\n",
+    "                  2) Build candidate code by concatenating example['prompt'] + candidate solution.\n",
+    "                  3) Execute the test with `target` and candidate in the sandbox via `check_solution`.\n",
+    "\n",
+    "          - Aggregate per-test pass/fail into a single boolean for the example.\n",
+    "          - Compute pass@k across candidates.\n",
+    "        \"\"\"\n",
+    "        prediction, label = self._check_evaluation_inputs(prediction, label)\n",
+    "\n",
+    "        results = []\n",
+    "        for solution in prediction:\n",
+    "            # Each `label` item corresponds to the SAME example in our usage (benchmark runs per example),\n",
+    "            # but we preserve the structure consistent with the base class.\n",
+    "            solution_states = []\n",
+    "            for label_data in label:\n",
+    "                task_id = label_data[\"task_id\"]\n",
+    "                entry_point = label_data[\"entry_point\"]\n",
+    "                tests = label_data.get(\"tests\", [])\n",
+    "                imports = label_data.get(\"imports\", \"\")\n",
+    "                canonical_solution = label_data.get(\"canonical_solution\", \"\")\n",
+    "\n",
+    "                # Build reference env for computing `target`\n",
+    "                ref_ns = self._build_reference_namespace(imports=imports, canonical_solution=canonical_solution)\n",
+    "\n",
+    "                # Build candidate code (prompt + solution)\n",
+    "                prompt = self.get_example_by_id(task_id)[\"prompt\"]\n",
+    "                candidate_code = prompt + \"\\n\" + solution\n",
+    "\n",
+    "                # Run each test individually; any failure => whole example fails\n",
+    "                all_ok = True\n",
+    "                for raw_test in tests if tests else [\"# no tests provided\\nassert True, True\"]:\n",
+    "                    # Heuristically precompute `target`\n",
+    "                    exprs = self._extract_candidate_exprs_from_test(raw_test)\n",
+    "                    try:\n",
+    "                        target_value = self._compute_target_list(exprs, ref_ns)\n",
+    "                    except Exception as e:\n",
+    "                        # If we cannot compute target from the reference, fall back to True\n",
+    "                        logger.warning(f\"[SciCode] Fallback target=True for {task_id} due to: {e}\")\n",
+    "                        target_value = True\n",
+    "\n",
+    "                    # Compose a runnable unit-test block:\n",
+    "                    # We inject `imports`, bind `target`, then execute the original test code.\n",
+    "                    unit_test = (\n",
+    "                        (imports or \"\")\n",
+    "                        + \"\\n\"\n",
+    "                        + \"target = __TARGET_VALUE__\\n\"\n",
+    "                        + raw_test\n",
+    "                    )\n",
+    "\n",
+    "                    # Because `check_solution` runs code in separate exec, we stringify the target safely.\n",
+    "                    # We'll register a placeholder and pass the real object via the executor's globals.\n",
+    "                    # Our base framework doesn't support direct object injection; so we serialize small types.\n",
+    "                    # For numpy arrays/tuples we rely on repr + eval. If that fails, we degrade to boolean.\n",
+    "                    try:\n",
+    "                        # Light-weight serializer for numpy arrays / tuples / lists / scalars\n",
+    "                        def _pyrepr(obj):\n",
+    "                            if isinstance(obj, np.ndarray):\n",
+    "                                return f\"np.array({repr(obj.tolist())})\"\n",
+    "                            return repr(obj)\n",
+    "\n",
+    "                        unit_test = unit_test.replace(\n",
+    "                            \"__TARGET_VALUE__\", _pyrepr(target_value)\n",
+    "                        )\n",
+    "                    except Exception:\n",
+    "                        unit_test = unit_test.replace(\"__TARGET_VALUE__\", \"True\")\n",
+    "\n",
+    "                    # Optional special-case patching hook\n",
+    "                    candidate_code_patched, unit_test_patched = self.handle_special_cases(\n",
+    "                        task_id=task_id, solution=candidate_code, test=unit_test\n",
+    "                    )\n",
+    "\n",
+    "                    # Execute\n",
+    "                    state, message = self.check_solution(\n",
+    "                        task_id=task_id,\n",
+    "                        solution=candidate_code_patched,\n",
+    "                        test=unit_test_patched,\n",
+    "                        entry_point=entry_point,\n",
+    "                    )\n",
+    "                    if state != self.SUCCESS:\n",
+    "                        all_ok = False\n",
+    "                        break\n",
+    "\n",
+    "                solution_states.append(self.SUCCESS if all_ok else self.FAILURE)\n",
+    "            results.append(len(solution_states) == len(label) and all(s == self.SUCCESS for s in solution_states))\n",
+    "\n",
+    "        k_list = [self.k] if isinstance(self.k, int) else self.k\n",
+    "        pass_at_k = self.compute_pass_at_k(results, k_list)\n",
+    "        return pass_at_k\n",
+    "\n",
+    "\n",
+    "class AFlowSciCode(SciCode):\n",
+    "    \"\"\"\n",
+    "    AFlow-specific implementation of SciCode benchmark.\n",
+    "    Uses AFLOW_DATASET_FILES_MAP['scicode'] for split files (if provided by your distribution).\n",
+    "    \"\"\"\n",
+    "\n",
+    "    def __init__(self, path: str = None, mode: str = \"all\", timeout: int = 60, k: Union[int, list] = 1, **kwargs):\n",
+    "        path = os.path.expanduser(path or \"~/.evoagentx/data/aflow/scicode\")\n",
+    "        super().__init__(path=path, mode=mode, timeout=timeout, k=k, **kwargs)\n",
+    "\n",
+    "    def _load_data_from_file(self, file_name: str):\n",
+    "        if file_name is None:\n",
+    "            return None\n",
+    "        file_path = os.path.join(self.path, file_name)\n",
+    "        if not os.path.exists(file_path):\n",
+    "            logger.info(\"Downloading AFlow SciCode split files ...\")\n",
+    "            download_aflow_benchmark_data(dataset=\"scicode\", save_folder=self.path)\n",
+    "        return load_json(path=file_path, type=\"jsonl\")\n",
+    "\n",
+    "    def _load_data(self):\n",
+    "        # Prefer AFLOW split files when available; otherwise fall back to raw download.\n",
+    "        if \"scicode\" not in AFLOW_DATASET_FILES_MAP:\n",
+    "            logger.warning(\"AFLOW_DATASET_FILES_MAP has no entry for 'scicode'; falling back to raw SciCode jsonl.\")\n",
+    "            return super()._load_data()\n",
+    "\n",
+    "        splits = AFLOW_DATASET_FILES_MAP[\"scicode\"]\n",
+    "        data_all: Dict[str, List[Dict[str, Any]]] = {}\n",
+    "\n",
+    "        for split in (\"train\", \"dev\", \"test\"):\n",
+    "            fname = splits.get(split)\n",
+    "            if fname:\n",
+    "                logger.info(f\"Loading {split} data from {fname}\")\n",
+    "                raw_split = self._load_data_from_file(file_name=fname)\n",
+    "                # Normalize rows to examples\n",
+    "                examples: List[Dict[str, Any]] = []\n",
+    "                for row in raw_split or []:\n",
+    "                    try:\n",
+    "                        examples.extend(_coerce_scicode_row_to_examples(row))\n",
+    "                    except Exception as e:\n",
+    "                        logger.warning(f\"[AFlowSciCode] Skipping a malformed row in {split} due to: {e}\")\n",
+    "                data_all[split] = examples\n",
+    "            else:\n",
+    "                data_all[split] = None\n",
+    "\n",
+    "        if self.mode in (\"train\", \"all\"):\n",
+    "            self._train_data = data_all.get(\"train\")\n",
+    "        if self.mode in (\"dev\", \"all\"):\n",
+    "            self._dev_data = data_all.get(\"dev\")\n",
+    "        if self.mode in (\"test\", \"all\"):\n",
+    "            self._test_data = data_all.get(\"test\")\n",
+    "\n",
+    "    async def async_evaluate(self, graph: Callable, example: Any) -> float:\n",
+    "        \"\"\"\n",
+    "        Generate a solution asynchronously and return pass@1 for the example.\n",
+    "        \"\"\"\n",
+    "        prompt, entry_point = example[\"prompt\"], example[\"entry_point\"]\n",
+    "        solution = await graph(prompt, entry_point)\n",
+    "        label = self._get_label(example)\n",
+    "        metrics = await super().async_evaluate(prediction=solution, label=label)\n",
+    "        return metrics.get(\"pass@1\", 0.0)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a2bca001",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.0"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

evoagentx/benchmark/README.md

@@ -0,0 +1,178 @@
+# Benchmark 
+
+## Benchmark Overview 
+
+This repository provides a set of benchmarks to facilitate the evaluation of different agent-based systems. Below is a summary of the benchmarks currently included, along with basic dataset statistics: 
+
+
+| Task                      | Dataset Name    | # Train   | # Dev   | # Test |
+| ------------------------- | --------------- | --------- | ------- | ------ |
+| QA                        | NQ              | 79,168    | 8,757   | 3,610  |
+| Multi-Hop QA              | HotPotQA        | 90,447    | 7,405   | /      |
+| Math                      | GSM8K           | 7,473     | /       | 1,319  |
+| Math                      | MATH            | 7,500     | /       | 5,000  |
+| Code Generation           | HumanEval       | /         | /       | 164    |
+| Code Generation           | MBPP            | /         | /       | 427    |
+| Code Generation           | LiveCodeBench(v1~v5) | /         | /       | 400~880    |
+| Code Execution            | LiveCodeBench   | /         | /       | 479      |
+| Test Output Prediction    | LiveCodeBench   | /         | /       | 442      |
+
+
+Below, we introduce the preprocessing steps and evaluation metrics for each benchmark. 
+
+- [Question Answering](#question-answering)
+  - [NQ](#nq)
+  - [HotPotQA](#hotpotqa)
+- [Math](#math)
+  - [GSM8K](#gsm8k)
+  - [MATH](#math)
+- [Code Generation](#code-generation)
+  - [HumanEval](#humaneval)
+  - [MBPP](#mbpp)
+  - [LiveCodeBench](#livecodebench)
+
+## Preprocessing and Evaluation Metrics 
+
+### Question Answering 
+
+For the QA datasets, we use Exact Match (EM), F1, and Accuracy (ACC) as evaluation metrics by default. EM requires the predicted answer to be exactly the same as the ground truth answer. ACC requires that the predicted answer contains the ground-truth answer, which is useful when the LLM is used to generate the answer. 
+
+#### NQ
+[Natural Questions (NQ)](https://github.com/google-research-datasets/natural-questions) contains questions from the Google search engine and the answers, annotated by human annotators, are paragraphs or entities in the Wikipedia page of the top 5 search results. We use the dataset splits provided by the [DPR](https://github.com/facebookresearch/DPR) repository, which contains 79,168 training, 8,757 development, and 3,610 test examples. 
+
+You can load the dataset using the following code: 
+```python
+from evoagentx.benchmark import NQ
+nq_dataset = NQ() # optional: path="/path/to/save_data"
+test_data = nq_dataset.get_test_data()
+```
+Each example in the dataset is in the following format: 
+```json
+{
+    "id": "test-1", 
+    "question": "the question", 
+    "answers": ["possible answers"]
+}
+```
+
+
+#### HotPotQA 
+[HotPotQA](https://hotpotqa.github.io/) is a multi-hop QA dataset that requires multi-step reasoning to answer the question. We use the distractor setting of the dataset. Each example contains a question, an answer, some context that contians both supporting and distractor information, and supporting facts. We only include the training and development sets, as the test set is not publicly available. 
+
+You can load the dataset using the following code: 
+```python
+from evoagentx.benchmark import HotPotQA
+hotpotqa_dataset = HotPotQA() # optional: path="/path/to/save_data"
+test_data = hotpotqa_dataset.get_test_data()
+```
+Each example in the dataset is in the following format, where the second element (int) of a supporting_fact is the index of the sentence in the context that supports the answer. 
+```json
+{
+        "_id": "the id of the example", 
+        "question": "the question", 
+        "answer": "the answer", 
+        "context": [["context_title", ["context_sentence", "another_sentence"]]],
+        "supporting_facts": [["supporting_title", 0]]
+    }
+```
+
+
+### Math
+
+For match datasets, we use the solve rate as the evaluation metric. The solve rate is the ratio of the number of examples that are solved correctly to the total number of examples.
+
+#### GSM8K 
+[GSM8K](https://github.com/openai/grade-school-math) consists of high quality grade school math problems created by human problem writers. These problems require multi-step mathematical reasoning to solve. We use the dataset splits provided by the original repository, which contains 7.5K training problems and 1K test problems. 
+
+You can load the dataset using the following code: 
+```python
+from evoagentx.benchmark import GSM8K
+gsm8k_dataset = GSM8K() # optional: path="/path/to/save_data"
+test_data = gsm8k_dataset.get_test_data()
+```
+Each example in the dataset is in the following format: 
+```json
+{
+    "id": "test-1", 
+    "question": "the question", 
+    "answer": "the answer"
+}
+```
+
+#### MATH 
+The [Mathematics Aptitude Test of Heuristics (MATH)](https://github.com/hendrycks/math) dataset consists of problems from mathematics competitions, including the AMC 10, AMC 12, AIME, etc. Each problem in MATH has a step-by-step solution. We use the dataset splits provided by the original repository, which contains 7.5K training problems and 5K test problems. 
+
+You can load the dataset using the following code: 
+```python
+from evoagentx.benchmark import MATH
+math_dataset = MATH() # optional: path="/path/to/save_data"
+test_data = math_dataset.get_test_data()
+```
+Each example in the dataset is in the following format. For the `level` field, valid values are: "Level 1", "Level 2", "Level 3", "Level 4", "Level 5", and "Level ?". The `type` field can be one of: "Geometry", "Algebra", "Intermediate Algebra", "Counting & Probability", "Precalculus", "Number Theory", or "Prealgebra".
+
+```json
+{
+    "id": "test-1", 
+    "problem": "the problem", 
+    "solution": "the solution",
+    "level": "Level 1",
+    "type": "Algebra"
+}
+```
+
+### Code Generation 
+For the code generation benchmarks, we use pass@k as the evaluation metric, where k is the number of solutions for each problem. By default, k is set to 1. 
+
+#### HumanEval 
+[HumanEval](https://github.com/openai/human-eval) is a dataset of 164 coding problems from the HumanEval benchmark. Each problem contains a function signature, a canonical solution, and a set of unit tests. 
+
+You can load the dataset using the following code: 
+```python
+from evoagentx.benchmark import HumanEval
+humaneval_dataset = HumanEval() # optional: path="/path/to/save_data"
+test_data = humaneval_dataset.get_test_data()
+```
+Each example in the dataset is in the following format: 
+```json
+{
+    "task_id": "HumanEval/0", 
+    "prompt": "the prompt of the problem", 
+    "entry_point": "the name of the function to be tested",
+    "canonical_solution": "the canonical solution of the problem", 
+    "test": "the unit tests of the problem"
+}
+```
+
+#### MBPP 
+[Mostly Basic Python Problems (MBPP)](https://github.com/google-research/google-research/tree/master/mbpp) consists of hundreds of entry-level Python programming problems. Each problem consists of a task description, code solution and 3 automated test cases. We use the [sanitized subset](https://github.com/google-research/google-research/blob/master/mbpp/sanitized-mbpp.json) of the MBPP dataset, which consists of 427 problems with data that are hand-verfied by the authors. To facilitate the evaluation, we convert the MBPP dataset into the HumanEval format. 
+
+You can load the dataset using the following code: 
+```python
+from evoagentx.benchmark import MBPP
+mbpp_dataset = MBPP() # optional: path="/path/to/save_data"
+test_data = mbpp_dataset.get_test_data()
+```
+Each example in the dataset is in the following format, where we keep the original MBPP `task_id`.  
+```json
+{
+    "task_id": 2, 
+    "prompt": "the prompt of the problem", 
+    "entry_point": "the name of the function to be tested",
+    "canonical_solution": "the canonical solution of the problem", 
+    "test": "the unit tests of the problem"
+}
+```
+You can also access the original MBPP attributes such as "code", "test_list" in the example by using `example["code"]`. 
+
+
+#### LiveCodeBench 
+[LiveCodeBench](https://livecodebench.github.io/) is a contamination-free evaluation benchmark of LLMs for code that continuously collects new problems over time. Particularly, LiveCodeBench also focuses on broader code-related capabilities, such as code execution, and test output prediction, beyond mere code generation. Currently, LiveCodeBench hosts over three hundred high-quality coding problems published between May 2023 and February 2024. 
+
+You can load the dataset using the following code, where `scenario` can be one of [`code_generation`, `test_output_prediction`, `code_execution`] indicating different tasks. `version` denotes different versions of the code generation datasets, which is only available for `code_generation` scenario, and can be one of `["release_v1", "release_v2", "release_v3", "release_v4", "release_v5", "release_latest"]`. Please refer to the [LiveCodeBench](https://livecodebench.github.io/) repository for more details. 
+
+```python
+from evoagentx.benchmark import LiveCodeBench
+livecodebench_dataset = LiveCodeBench(scenario="code_generation", version="release_v1") # optional: path="/path/to/save_data"
+test_data = livecodebench_dataset.get_test_data() 
+```
+

evoagentx/benchmark/Untitled.ipynb

@@ -0,0 +1,6 @@
+{
+ "cells": [],
+ "metadata": {},
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

evoagentx/benchmark/WorfBench.py

@@ -0,0 +1,155 @@
+import os
+import json
+import random
+from typing import Any, Dict, Callable, List
+from .benchmark import Benchmark
+from .measures import exact_match_score, f1_score, acc_score
+from ..core.logging import logger
+from ..core.module_utils import load_json
+from datasets import load_dataset
+
+# WorfBench dataset file mapping
+WORFBENCH_FILES_MAP = {
+    "train": "worfbench_train.json",
+    "test": "worfbench_test.json"
+}
+VALID_WORFBENCH_FILES = list(WORFBENCH_FILES_MAP.values())
+
+def evaluate_workflow_sequence(prediction: List[Any], ground_truth: List[Any]) -> float:
+    """Evaluate F1 score for sequence workflow."""
+    from .measures import f1_score
+    return f1_score(prediction=prediction, ground_truth=ground_truth)
+
+def evaluate_workflow_graph(prediction: Dict[str, Any], ground_truth: Dict[str, Any]) -> float:
+    """Evaluate F1 score for graph workflow."""
+    pred_nodes = set(prediction.get("nodes", []))
+    true_nodes = set(ground_truth.get("nodes", []))
+    pred_edges = set(tuple(edge) for edge in prediction.get("edges", []))
+    true_edges = set(tuple(edge) for edge in ground_truth.get("edges", []))
+    
+    node_precision = len(pred_nodes & true_nodes) / len(pred_nodes) if pred_nodes else 0
+    node_recall = len(pred_nodes & true_nodes) / len(true_nodes) if true_nodes else 0
+    edge_precision = len(pred_edges & true_edges) / len(pred_edges) if pred_edges else 0
+    edge_recall = len(pred_edges & true_edges) / len(true_edges) if true_edges else 0
+    
+    node_f1 = 2 * (node_precision * node_recall) / (node_precision + node_recall) if (node_precision + node_recall) > 0 else 0
+    edge_f1 = 2 * (edge_precision * edge_recall) / (edge_precision + edge_recall) if (edge_precision + edge_recall) > 0 else 0
+    
+    return (node_f1 + edge_f1) / 2
+
+def download_worfbench_data(dataset: str, save_folder: str) -> None:
+    """
+    Download WorfBench dataset from Hugging Face.
+
+    Args:
+        dataset (str): Dataset name ("worfbench").
+        save_folder (str): Directory to save data.
+    """
+    datasets_map = {
+        "train": {"repo_id": "zjunlp/WorFBench_train", "filename": "worfbench_train.json", "split": "train"},
+        "test": {"repo_id": "zjunlp/WorFBench_test", "filename": "worfbench_test.json", "split": "test"}
+    }
+    
+    os.makedirs(save_folder, exist_ok=True)
+    for split, info in datasets_map.items():
+        repo_id = info["repo_id"]
+        filename = info["filename"]
+        dataset_split = info["split"]
+        save_path = os.path.join(save_folder, filename)
+        
+        if not os.path.exists(save_path):
+            logger.info(f"Downloading {split} split of {dataset} from {repo_id}...")
+            try:
+                # Load dataset
+                ds = load_dataset(repo_id, split=dataset_split)
+                # Convert dataset to list and save as JSON
+                data = [item for item in ds]
+                with open(save_path, 'w', encoding='utf-8') as f:
+                    json.dump(data, f, ensure_ascii=False, indent=2)
+                logger.info(f"Successfully downloaded and saved {filename} to {save_path}")
+            except Exception as e:
+                logger.error(f"Failed to download or save {filename}: {e}")
+                raise
+        else:
+            logger.info(f"File {save_path} already exists, skipping download.")
+
+class WorfBench(Benchmark):
+    """
+    WorfBench evaluation class for assessing LLM agents on complex workflow generation tasks.
+    Assumed data structure:
+    {
+        "id": str,
+        "task": str,
+        "context": list of dicts (e.g., [{"title": str, "content": list of str}]),
+        "expected_output": str or dict (sequence or graph),
+        "type": str,
+        "level": str
+    }
+    """
+    def __init__(self, path: str = None, mode: str = "test", **kwargs):
+        path = os.path.expanduser(path or "~/.worfbench/data")
+        super().__init__(name=type(self).__name__, path=path, mode=mode, **kwargs)
+
+    def _load_data_from_file(self, file_name: str) -> Dict:
+        if file_name is None:
+            return None
+        file_path = os.path.join(self.path, file_name)
+        if not os.path.exists(file_path):
+            download_worfbench_data(dataset="worfbench", save_folder=self.path)
+        if not os.path.exists(file_path):
+            logger.error(f"File {file_path} still does not exist after download attempt!")
+            return None
+        logger.info(f"Loading WorfBench data from {file_path} ...")
+        data = load_json(path=file_path, type="json")
+        if data is None:
+            logger.error(f"Failed to load data from {file_path}")
+            return None
+        return data
+
+    def _load_data(self) -> None:
+        if self.mode in ["train", "dev"]:
+            self._train_data = self._load_data_from_file(file_name=WORFBENCH_FILES_MAP["train"])
+            if self.mode == "dev":
+                if self._train_data:
+                    random.seed(42)
+                    keys = list(self._train_data.keys())
+                    n_dev = len(self._train_data[keys[0]]) // 10 or 1
+                    indices = list(range(len(self._train_data[keys[0]])))
+                    random.shuffle(indices)
+                    self._train_data = {k: [v[i] for i in indices[:n_dev]] for k, v in self._train_data.items()}
+        if self.mode == "test":
+            self._test_data = self._load_data_from_file(file_name=WORFBENCH_FILES_MAP["test"])
+
+    def _get_label(self, example: Dict) -> Any:
+        return example.get("expected_output", "")
+
+    def _get_id(self, example: Dict) -> Any:
+        return example.get("id", "")
+
+    def evaluate(self, prediction: Any, label: Any) -> Dict:
+        if isinstance(prediction, list) and isinstance(label, list):
+            f1 = evaluate_workflow_sequence(prediction, label)
+        elif isinstance(prediction, dict) and isinstance(label, dict):
+            f1 = evaluate_workflow_graph(prediction, label)
+        else:
+            f1 = f1_score(prediction=str(prediction), ground_truth=str(label))
+        em = exact_match_score(prediction=prediction, ground_truth=label)
+        acc = acc_score(prediction=prediction, ground_truths=[label])
+        return {"em": em, "f1": f1, "acc": acc}
+
+    async def async_evaluate(self, graph: Callable, example: Dict) -> float:
+        task = example.get("task", "")
+        context = "\n".join(
+            f"{ctx.get('title', '')}: {' '.join(ctx.get('content', []))}"
+            for ctx in example.get("context", [])
+            if isinstance(ctx, dict)
+        )
+        inputs = f"Task: {task}\nContext: {context}\nGenerate workflow:\nAnswer:"
+        try:
+            generated_workflow = await graph(inputs)
+        except Exception as e:
+            logger.error(f"Error generating workflow: {e}")
+            generated_workflow = ""
+        label = self._get_label(example)
+        metrics = self.evaluate(prediction=generated_workflow, label=label)
+        return metrics["f1"]