{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"id": "4c2a6fa7",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/gpfs/radev/home/tl688/.conda/envs/evoagentx/lib/python3.11/site-packages/PyPDF2/__init__.py:21: DeprecationWarning: PyPDF2 is deprecated. Please move to the pypdf library instead.\n",
" warnings.warn(\n"
]
}
],
"source": [
"import os\n",
"\n",
"from dotenv import load_dotenv\n",
"\n",
"from evoagentx.agents.agent_manager import AgentManager\n",
"from evoagentx.benchmark import HotPotQA\n",
"from evoagentx.core.callbacks import suppress_logger_info\n",
"from evoagentx.core.logging import logger\n",
"from evoagentx.evaluators import Evaluator\n",
"from evoagentx.models import OpenAILLM, OpenAILLMConfig\n",
"from evoagentx.optimizers import TextGradOptimizer\n",
"from evoagentx.prompts import StringTemplate\n",
"from evoagentx.workflow import SequentialWorkFlowGraph\n",
"from dotenv import load_dotenv\n",
"\n",
"from evoagentx.agents.agent_manager import AgentManager\n",
"from evoagentx.benchmark import MBPP\n",
"from evoagentx.core.callbacks import suppress_logger_info\n",
"from evoagentx.core.logging import logger\n",
"from evoagentx.evaluators import Evaluator\n",
"from evoagentx.models import OpenAILLM, OpenAILLMConfig\n",
"from evoagentx.optimizers import QASTRUCTUREOptimizer, TextGradOptimizer\n",
"from evoagentx.prompts import StringTemplate\n",
"from evoagentx.workflow import SequentialWorkFlowGraph\n",
"\n",
"from evoagentx.models import OpenAILLMConfig, OpenAILLM\n",
"from evoagentx.workflow import SEWWorkFlowGraph, QASTRUCTUREWorkFlowGraph\n",
"from evoagentx.agents import AgentManager\n",
"from evoagentx.benchmark import HumanEval,AFlowMBPP\n",
"from evoagentx.evaluators import Evaluator \n",
"from evoagentx.optimizers import SEWOptimizer, STRUCTUREOptimizer\n",
"from evoagentx.optimizers.structure_optimizer import STRUCTUREWorkFlowScheme\n",
"from evoagentx.core.callbacks import suppress_logger_info\n",
"\n",
"from evoagentx.models import OpenAILLMConfig, OpenAILLM,AzureOpenAIConfig,LiteLLMConfig,LiteLLM\n",
"from evoagentx.workflow import SEWWorkFlowGraph \n",
"from evoagentx.agents import AgentManager\n",
"from evoagentx.benchmark import MBPPPLUS, AFlowMBPPPLUS\n",
"from evoagentx.evaluators import Evaluator \n",
"from evoagentx.optimizers import SEWOptimizer \n",
"from evoagentx.core.callbacks import suppress_logger_info\n",
"from evoagentx.benchmark import HumanEvalPLUS\n",
"from evoagentx.benchmark import SciCode\n",
"from copy import deepcopy\n",
"\n",
"api_key = \"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\"\n",
"# Function to encode the image\n",
"import os\n",
"os.environ[\"OPENAI_API_KEY\"] = api_key\n",
"OPENAI_API_KEY = os.getenv(\"OPENAI_API_KEY\")\n",
"\n",
"import nest_asyncio\n",
"nest_asyncio.apply()\n",
"\n",
"class HotPotQASplits(HotPotQA):\n",
"\n",
" def _load_data(self):\n",
" # load the original test data \n",
" super()._load_data()\n",
" # split the data into train, dev and test\n",
" import numpy as np \n",
" np.random.seed(42)\n",
" permutation = np.random.permutation(len(self._dev_data))\n",
" full_test_data = self._dev_data \n",
" # randomly select 10 samples for train, 40 for dev, and 100 for test\n",
" self._train_data = [full_test_data[idx] for idx in permutation[:50]]\n",
" self._dev_data = [full_test_data[idx] for idx in permutation[:50]]\n",
" self._dev_data_full = deepcopy(self._dev_data)\n",
" self._test_data = [full_test_data[idx] for idx in permutation[50:500]]\n",
" self._fulldata = full_test_data\n",
"\n",
"\n",
"def collate_func(example: dict) -> dict:\n",
" context_list = []\n",
" for item in example[\"context\"]:\n",
" context = \"Title: {}\\nText: {}\".format(item[0], \" \".join([t.strip() for t in item[1]]))\n",
" context_list.append(context)\n",
" context = \"\\n\\n\".join(context_list)\n",
" problem = \"Context: {}\\n\\nQuestion: {}\\n\\nAnswer:\".format(context, example[\"question\"])\n",
" return {\"question\": problem}\n",
"\n",
"\n",
"# hotpotqa_graph_data = {\n",
"# \"goal\": \"Answer user questions accurately and concisely by decomposing the problem into analysis, retrieval, drafting, critique, and refinement.\",\n",
"# \"tasks\": [\n",
"# {\n",
"# \"name\": \"analyze_question\",\n",
"# \"description\": \"Normalize and analyze the question to clarify intent, extract key entities, and identify the information need.\",\n",
"# \"inputs\": [\n",
"# {\"name\": \"question\", \"type\": \"str\", \"required\": True,\n",
"# \"description\": \"The original question from the user.\"},],\n",
"# \"outputs\": [\n",
"# {\"name\": \"normalized_question\", \"type\": \"str\", \"required\": True,\n",
"# \"description\": \"Rephrased, unambiguous version of the question.\"}\n",
"# ],\n",
"# \"prompt_template\": StringTemplate(instruction=\"Think step by step to answer the question. You should explain your thinking process in the 'thought' field, and provide the final answer in the 'answer' field.\\nFormat your output in xml format, such as xxx and xxx.\"),\n",
"# \"parse_mode\": \"xml\"\n",
"# },\n",
"\n",
"# {\n",
"# \"name\": \"generate_answer\",\n",
"# \"description\": \"Draft an answer using the retrieved context.\",\n",
"# \"inputs\": [\n",
"# {\"name\": \"normalized_question\", \"type\": \"str\", \"required\": True,\n",
"# \"description\": \"Normalized question to be answered.\"}\n",
"# ],\n",
"# \"outputs\": [\n",
"# {\"name\": \"draft_answer\", \"type\": \"str\", \"required\": True,\n",
"# \"description\": \"A concise, direct answer to the question.\"}\n",
"# ],\n",
"# \"prompt_template\": StringTemplate(instruction=\"Think step by step to answer the question. You should explain your thinking process in the 'thought' field, and provide the final answer in the 'answer' field.\\nFormat your output in xml format, such as xxx and xxx.\"),\n",
"# \"parse_mode\": \"xml\"\n",
"# },\n",
"\n",
"# # {\n",
"# # \"name\": \"critique_answer\",\n",
"# # \"description\": \"Check the drafted answer for correctness, faithfulness to the context, and alignment with the question.\",\n",
"# # \"inputs\": [\n",
"# # {\"name\": \"normalized_question\", \"type\": \"str\", \"required\": True,\n",
"# # \"description\": \"The normalized question.\"},\n",
"# # {\"name\": \"draft_answer\", \"type\": \"str\", \"required\": True,\n",
"# # \"description\": \"The answer proposed by generate_answer.\"}\n",
"# # ],\n",
"# # \"outputs\": [\n",
"# # {\"name\": \"is_valid\", \"type\": \"bool\", \"required\": True,\n",
"# # \"description\": \"Whether the draft answer is correct and well-supported by the context.\"},\n",
"# # {\"name\": \"issues\", \"type\": \"list[str]\", \"required\": False,\n",
"# # \"description\": \"List of detected issues (e.g., hallucination, missing details, ambiguity).\"},\n",
"# # {\"name\": \"suggested_improvements\", \"type\": \"str\", \"required\": False,\n",
"# # \"description\": \"Textual suggestions on how to improve the answer, if needed.\"}\n",
"# # ],\n",
"# # \"prompt_template\": StringTemplate(\n",
"# # instruction=(\n",
"# # \"You are an Answer Critique Agent.\\n\"\n",
"# # \"Compare the draft answer to the context and normalized question.\\n\"\n",
"# # \"1) Mark if the answer is fully supported and correctly addresses the question.\\n\"\n",
"# # \"2) If not valid, list concrete issues and suggest how to fix them.\\n\"\n",
"# # \"Return XML with , (items as ), and .\"\n",
"# # )\n",
"# # ),\n",
"# # \"parse_mode\": \"xml\"\n",
"# # },\n",
"\n",
"# # {\n",
"# # \"name\": \"refine_answer\",\n",
"# # \"description\": \"Refine or rewrite the answer based on critique, preserving factual alignment with the context.\",\n",
"# # \"inputs\": [\n",
"# # {\"name\": \"normalized_question\", \"type\": \"str\", \"required\": True,\n",
"# # \"description\": \"The normalized question.\"},\n",
"# # {\"name\": \"draft_answer\", \"type\": \"str\", \"required\": True,\n",
"# # \"description\": \"The initial answer to be refined.\"},\n",
"# # {\"name\": \"is_valid\", \"type\": \"bool\", \"required\": True,\n",
"# # \"description\": \"Validation flag from critique_answer.\"},\n",
"# # {\"name\": \"suggested_improvements\", \"type\": \"str\", \"required\": False,\n",
"# # \"description\": \"Guidance from the critique_answer step on how to improve the answer.\"}\n",
"# # ],\n",
"# # \"outputs\": [\n",
"# # {\"name\": \"answer\", \"type\": \"str\", \"required\": True,\n",
"# # \"description\": \"Final, concise, and validated answer ready to return to the user.\"}\n",
"# # ],\n",
"# # \"prompt_template\": StringTemplate(\n",
"# # instruction=(\n",
"# # \"You are an Answer Refinement Agent.\\n\"\n",
"# # \"If is_valid is true, you may lightly polish the draft answer for clarity and brevity.\\n\"\n",
"# # \"If is_valid is false, use the suggested improvements and context to rewrite the answer so that it is correct, \"\n",
"# # \"fully supported, and concise (1–3 sentences).\\n\"\n",
"# # \"Return XML with only.\"\n",
"# # )\n",
"# # ),\n",
"# # \"parse_mode\": \"xml\"\n",
"# # }\n",
"# ]\n",
"# }\n",
"hotpotqa_graph_data = {\n",
" \"goal\": \"Answer the question based on the context. The answer should be a direct response to the question, without including explanations or reasoning.\",\n",
" \"tasks\": [\n",
" {\n",
" \"name\": \"answer_generate\",\n",
" \"description\": \"Answer the question based on the context.\",\n",
" \"inputs\": [\n",
" {\"name\": \"question\", \"type\": \"str\", \"required\": True, \"description\": \"The problem to solve.\"}\n",
" ],\n",
" \"outputs\": [\n",
" {\"name\": \"answer\", \"type\": \"str\", \"required\": True, \"description\": \"The answer to the problem.\"}\n",
" ],\n",
" \"prompt_template\": StringTemplate(instruction=\"Think step by step to answer the question. You should explain your thinking process in the 'thought' field, and provide the final answer in the 'answer' field.\\nFormat your output in xml format, such as xxx and xxx.\"),\n",
" \"parse_mode\": \"xml\"\n",
" }\n",
" ] \n",
"}\n",
"\n",
"\n",
"# os.environ[\"AZURE_OPENAI_DEPLOYMENT_NAME\"] = \"gpt-4o-mini\"\n",
"# os.environ[\"AZURE_OPENAI_ENDPOINT\"] = \"https://optimizehumaneval.cognitiveservices.azure.com/\"\n",
"# os.environ[\"AZURE_OPENAI_KEY\"] = \"2b7h6anDXRsl5XHDUAGKHpjh3DLv9kLjcjGXN6PvsEmLVf1i3imMJQQJ99BKACYeBjFXJ3w3AAABACOGATqP\"\n",
"# os.environ[\"AZURE_OPENAI_API_VERSION\"] = \"2025-01-01-preview\"\n",
"# llm_config = LiteLLMConfig(model=\"azure/\" + os.getenv(\"AZURE_OPENAI_DEPLOYMENT_NAME\"), # Azure model format\n",
"# azure_endpoint=os.getenv(\"AZURE_OPENAI_ENDPOINT\"),\n",
"# azure_key=os.getenv(\"AZURE_OPENAI_KEY\"),\n",
"# api_version=os.getenv(\"AZURE_OPENAI_API_VERSION\", \"2024-12-01-preview\"), top_p=0.85, temperature=0.2, frequency_penalty=0.0, presence_penalty=0.0)\n",
"\n",
"# executor_llm = LiteLLM(config=llm_config)\n",
"# optimizer_llm = LiteLLM(config=llm_config)\n",
"# llm = executor_llm"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "ad0efa03",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"evoagentx.optimizers.sew_optimizer.SEWOptimizer"
]
},
"execution_count": 2,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"SEWOptimizer "
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "ad4b2024",
"metadata": {},
"outputs": [],
"source": [
"# difficult easy "
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "c95059f0",
"metadata": {},
"outputs": [],
"source": [
"from evoagentx.benchmark import HotPotQA"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "84efabfa",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"\u001b[32m2025-12-14 09:23:35.234\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.benchmark.hotpotqa\u001b[0m:\u001b[36m_load_data_from_file\u001b[0m:\u001b[36m51\u001b[0m - \u001b[1mloading HotPotQA data from /gpfs/radev/home/tl688/.evoagentx/data/hotpotqa/hotpot_train_v1.1.json ...\u001b[0m\n",
"\u001b[32m2025-12-14 09:23:38.932\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.benchmark.hotpotqa\u001b[0m:\u001b[36m_load_data_from_file\u001b[0m:\u001b[36m51\u001b[0m - \u001b[1mloading HotPotQA data from /gpfs/radev/home/tl688/.evoagentx/data/hotpotqa/hotpot_dev_distractor_v1.json ...\u001b[0m\n"
]
}
],
"source": [
"llm_config = OpenAILLMConfig(model=\"gpt-4o-mini-2024-07-18\", openai_key=OPENAI_API_KEY, top_p=0.85, temperature=0.2, frequency_penalty=0.0, presence_penalty=0.0)\n",
"llm = OpenAILLM(config=llm_config)\n",
"\n",
"# obtain SEW workflow \n",
"# sew_graph = SEWWorkFlowGraph.from_dict(hotpotqa_graph_data)\n",
"# agent_manager = AgentManager()\n",
"# agent_manager.add_agents_from_workflow(sew_graph, executor_llm.config)\n",
"# obtain SEW workflow \n",
"# sew_graph = QASTRUCTUREWorkFlowGraph.from_dict(hotpotqa_graph_data)\n",
"sew_graph = SequentialWorkFlowGraph.from_dict(hotpotqa_graph_data)\n",
"agent_manager = AgentManager()\n",
"agent_manager.add_agents_from_workflow(sew_graph, llm_config=llm_config)\n",
"benchmark = HotPotQASplits()\n",
"# obtain Evaluator\n",
"evaluator = Evaluator(\n",
" llm=llm, \n",
" agent_manager=agent_manager, \n",
" collate_func=collate_func, \n",
" num_workers=20, \n",
" verbose=True\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "543936f2",
"metadata": {
"scrolled": true
},
"outputs": [
{
"data": {
"text/plain": [
"{'_id': '5ae6316d5542996de7b71b87',\n",
" 'answer': 'Ricard Rubio i Vives',\n",
" 'question': 'Which FC Barcelona signee was a contender for the Rookie of the Year Award when he played for the Timberwolves?',\n",
" 'supporting_facts': [['2011–12 Minnesota Timberwolves season', 0],\n",
" ['2011–12 Minnesota Timberwolves season', 2],\n",
" ['Ricky Rubio', 0],\n",
" ['Ricky Rubio', 6]],\n",
" 'context': [['Emilio Sagi Liñán',\n",
" ['Emilio Sagi Liñán (born Bolívar, Buenos Aires, Argentina, 15 March 1900; died Barcelona, 25 May 1951), was a former Spanish footballer who played as a left-winger for FC Barcelona, the Catalan XI and Spain during the 1920s and 1930s.',\n",
" \" He was the son of Emilio Sagi Barba, the Catalan baritone singer, and Concepción Liñán Pelegrí, a dancer, and as a result, was widely referred to as Sagibarba (father's surnames together in a single surname).\",\n",
" ' During his playing career he played 455 games and scored 134 goals for FC Barcelona and is best remembered for forming a successful partnership with Paulino Alcántara.',\n",
" ' Together with Josep Samitier, Ricardo Zamora, Félix Sesúmaga and, later, Franz Platko they were prominent members of the successful FC Barcelona team coached by Jack Greenwell.',\n",
" ' His younger brother, Luís Sagi Vela, followed in his fathers footsteps and also became a successful baritone singer.',\n",
" ' His son, Victor Sagi, later ran one of the biggest advertising agencies in Spain and in 1978 announced his candidacy for the presidency of FC Barcelona, but withdrew before the election was held.']],\n",
" ['Ricky Rubio',\n",
" ['Ricard Rubio i Vives (born October 21, 1990) is a Spanish professional basketball player for the Utah Jazz of the National Basketball Association (NBA).',\n",
" ' Rubio became the youngest player ever to play in the Spanish ACB League on October 15, 2005, at age 14.',\n",
" ' He made his EuroLeague debut on October 24, 2006, at age 16, becoming the first player born in the 1990s to play in a EuroLeague game.',\n",
" ' He is the fifth-youngest player to make their debut in the EuroLeague.',\n",
" ' On June 25, 2009, he was drafted with the fifth pick in the first round of the 2009 NBA draft by the Timberwolves, making him the first player born in the 1990s to be drafted by the NBA.',\n",
" ' The Timberwolves had an agreement in principle with his former Spanish team, DKV Joventut, to buy out his contract, but Rubio backed out of the deal.',\n",
" ' On August 31, 2009, Joventut traded the rights to Rubio to FC Barcelona, and Rubio signed a six-year contract with FC Barcelona the following day.',\n",
" ' In 2011, Rubio joined the Minnesota Timberwolves, and spent six seasons in Minnesota before being traded to the Jazz in June 2017.']],\n",
" ['Joaquim Peris de Vargas',\n",
" ['Joaquim Peris de Vargas is a former President of FC Barcelona.',\n",
" ' He was one of the most controversial Presidents in the history of FC Barcelona.',\n",
" ' He began his career as manager in 1910 as he occupied the vice presidency, a position he held with various presidents.',\n",
" ' Taking advantage of Pay Àlvar resignation in September 1914, Vargas Peris assumed leadership of FC Barcelona.',\n",
" ' His spell in charge at the club was marked by constant controversy, because I always wanted to impose his opinion and even got the players rebelling against him.',\n",
" ' Vargas was famous for his quote: \"I am Barcelona.\"',\n",
" ' He left the organization at the request of the general assembly of FC Barcelona and he was forced to resign at the end of the season 1914-15.']],\n",
" ['Ferenc Plattkó',\n",
" ['Ferenc Plattkó (born Franz Platko Kopiletz in Budapest, Hungary, 2 December 1898, died Santiago, Chile, 2 September 1983), also known as Ferenc Platko or Francisco Platko, was a Hungarian footballer and manager of Austrian origin.',\n",
" ' During the 1910s and 1920s he played as a goalkeeper for Vasas SC, WAC Vienna, KAFK Kula, MTK Hungária FC, FC Barcelona, Recreativo de Huelva.',\n",
" ' He subsequently worked as a coach in Europe and South America, most notably with FC Barcelona, Colo-Colo, River Plate, Boca Juniors and Chile.',\n",
" ' Platko was an early FC Barcelona legend and was a team-mate of Paulino Alcántara, Josep Samitier and Sagibarba.',\n",
" ' His bravery as a goalkeeper was immortalized by Rafael Alberti in the poem \"Oda A Platko\".',\n",
" ' After retiring as a player he returned to the club as a coach on two occasions (1934–35, 1955–56).']],\n",
" ['2011–12 Minnesota Timberwolves season',\n",
" ['The 2011–12 Minnesota Timberwolves season was the 23rd season of the franchise in the National Basketball Association (NBA).',\n",
" ' In their first season with head coach Rick Adelman, the team finished the lockout-shortened season with a 26–40 record, nine wins above their previous season and finished in 12th place in the Western Conference.',\n",
" ' This season saw the debut of 2009 draftee Ricky Rubio, who was a contender for the Rookie of the Year Award until he tore his ACL and his lateral collateral ligament after colliding into Kobe Bryant during a game against the Los Angeles Lakers and was out for the rest of the season.',\n",
" ' Following the season, Brad Miller retired.']],\n",
" ['Nou Palau Blaugrana',\n",
" ['The Nou Palau Blaugrana will be a multi-sports indoor arena, located in Barcelona, Catalonia, Spain.',\n",
" ' The arena will serve as the home arena for the basketball (FC Barcelona Bàsquet) and handball (FC Barcelona Handbol) sections of the multi-sports club FC Barcelona.',\n",
" ' The Nou Palau Blaugrana will have a capacity of 12,500 spectators.']],\n",
" ['Josep Maria Fusté',\n",
" ['Josep Maria Fusté Blanch (born 15 April 1941) is a retired Spanish footballer and captain of FC Barcelona during the 1960s and early 1970s.',\n",
" ' In 1964, together with Luis Suárez, Amancio Amaro, José Ángel Iribar and his FC Barcelona team mate, Jesús María Pereda, he also helped Spain win the European Championship.',\n",
" ' He also played for CA Osasuna and Hércules CF.',\n",
" ' After retiring as a player he worked as a public relations executive for \"Codorniu\", a Catalan sparkling wine company.',\n",
" ' He also served as president of the FC Barcelona veterans association and publicly supported Sixto Cambra, a Catalan nationalist, who stood against Josep Lluís Nuñez in the 1989 FC Barcelona presidential elections.']],\n",
" ['FC Barcelona Bàsquet B',\n",
" ['FC Barcelona Bàsquet B (English: FC Barcelona Basketball B), also currently known as FC Barcelona Lassa B for sponsorship reasons, is the reserve team of FC Barcelona Lassa.',\n",
" ' The team currently plays in the Spanish 2nd-tier level LEB Oro.']],\n",
" ['Enrique Fernández Viola',\n",
" ['Enrique Fernández Viola, commonly referred to as Enrique Fernández, (10 June 1912 – 6 October 1985) was a Uruguayan footballer and manager who played for Nacional, Talleres (RE), Independiente, FC Barcelona, Uruguay and the Catalan XI.',\n",
" ' As a manager, he won two Uruguayan championships with Nacional and La Liga titles with both FC Barcelona and Real Madrid.',\n",
" ' Along with Radomir Antic, he is one of only two coaches to have taken charge of both FC Barcelona and Real Madrid and he is the only coach to have won La Liga titles with both.',\n",
" ' He was born in Montevideo, Uruguay.']],\n",
" ['FC Barcelona Bàsquet',\n",
" ['FC Barcelona Bàsquet (English: FC Barcelona Basketball), also currently known as FC Barcelona Lassa for sponsorship reasons, is a Spanish professional basketball club.',\n",
" ' It is a part of the FC Barcelona multi sports club, and was founded on 24 August 1926, which makes it the oldest club in the Liga ACB.',\n",
" ' The club competes domestically in the Liga ACB and the EuroLeague.',\n",
" \" It has won seven of the last thirteen ACB championships, and in 2003, completed a Liga ACB (Spanish League), Copa del Rey (Spanish King's Cup) and EuroLeague triple crown.\",\n",
" ' FC Barcelona Bàsquet has played in seven EuroLeague Finals, with the last one being their 2010 win.']]],\n",
" 'type': 'bridge',\n",
" 'level': 'hard'}"
]
},
"execution_count": 7,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"benchmark._test_data[0]"
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "d2bba683",
"metadata": {},
"outputs": [],
"source": [
"# import json\n",
"# # with open(\"../../MaAS/maas/ext/maas/data/humaneval_train.jsonl\", 'w') as f:\n",
"# # json.dump(humaneval._dev_data, f, indent=2) # indent=4 makes the JSON output more readable\n",
"\n",
"\n",
"# # with open(\"../../MaAS/maas/ext/maas/data/humaneval_test.jsonl\", 'w') as f:\n",
"# # json.dump(humaneval._test_data, f, indent=2) # indent=4 makes the JSON output more readable\n",
"\n",
"# with open(\"../../MaAS/maas/ext/maas/data/humaneval_train.jsonl\", 'w') as f:\n",
"# for obj in humaneval._dev_data:\n",
"# json_line = json.dumps(obj)\n",
"# f.write(json_line + '\\n')\n",
" \n",
"# with open(\"../../MaAS/maas/ext/maas/data/humaneval_test.jsonl\", 'w') as f:\n",
"# for obj in humaneval._test_data:\n",
"# json_line = json.dumps(obj)\n",
"# f.write(json_line + '\\n')\n"
]
},
{
"cell_type": "code",
"execution_count": 9,
"id": "8598151b",
"metadata": {
"scrolled": false
},
"outputs": [
{
"data": {
"text/plain": [
"1"
]
},
"execution_count": 9,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"len(sew_graph.to_dict()['nodes'])"
]
},
{
"cell_type": "code",
"execution_count": 10,
"id": "b1f7fc18",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"0"
]
},
"execution_count": 10,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"len(sew_graph.edges)"
]
},
{
"cell_type": "code",
"execution_count": 11,
"id": "33859fa8",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"[]"
]
},
"execution_count": 11,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"sew_graph.edges"
]
},
{
"cell_type": "code",
"execution_count": 17,
"id": "227fc475",
"metadata": {
"scrolled": true
},
"outputs": [],
"source": [
"evaluator = Evaluator(llm=llm, agent_manager=agent_manager, collate_func=collate_func, num_workers=6, verbose=True)\n",
"# obtain SEWOptimizer after having more roles\n",
"optimizer = QASTRUCTUREOptimizer(\n",
" graph=sew_graph, \n",
" evaluator=evaluator, \n",
" llm=llm, \n",
" max_steps=20,\n",
" eval_rounds=1, \n",
" repr_scheme=\"python\", \n",
" optimize_mode=\"all\", \n",
" order=\"zero-order\",\n",
" max_rounds=1\n",
")\n",
"optimizer.calltime = 3\n",
"optimizer.collate_func = collate_func\n",
"\n",
"benchmark.error_list = {}\n",
"benchmark.timeout = 900\n",
"benchmark.dataname = 'hotpotqa'"
]
},
{
"cell_type": "code",
"execution_count": 49,
"id": "458eb432",
"metadata": {
"scrolled": true
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"Evaluating workflow: 0%| | 1/450 [00:01<14:12, 1.90s/it]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0.3333333333333333, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 0%| | 2/450 [00:03<13:59, 1.87s/it]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0.75, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 1%| | 3/450 [00:05<12:41, 1.70s/it]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 1%| | 4/450 [00:07<14:37, 1.97s/it]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 1%| | 5/450 [00:09<15:13, 2.05s/it]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 1%|▏ | 6/450 [00:11<15:15, 2.06s/it]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0.4, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 2%|▏ | 7/450 [00:14<15:53, 2.15s/it]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 2%|▏ | 8/450 [00:15<14:46, 2.01s/it]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 2%|▏ | 9/450 [00:17<13:13, 1.80s/it]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 2%|▏ | 10/450 [00:20<15:54, 2.17s/it]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0.18181818181818182, 'em': 0.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 2%|▏ | 11/450 [00:22<16:18, 2.23s/it]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 3%|▎ | 12/450 [00:25<16:39, 2.28s/it]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 3%|▎ | 13/450 [00:26<15:46, 2.17s/it]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 3%|▎ | 14/450 [00:29<15:57, 2.20s/it]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 3%|▎ | 15/450 [00:31<16:11, 2.23s/it]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 4%|▎ | 16/450 [00:33<16:03, 2.22s/it]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 4%|▍ | 17/450 [00:35<14:32, 2.02s/it]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 4%|▍ | 18/450 [00:37<15:17, 2.12s/it]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 4%|▍ | 19/450 [00:39<15:13, 2.12s/it]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 4%|▍ | 20/450 [00:42<15:28, 2.16s/it]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 5%|▍ | 21/450 [00:44<16:24, 2.29s/it]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0.7272727272727273, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 5%|▍ | 22/450 [00:46<14:59, 2.10s/it]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 5%|▌ | 23/450 [00:48<15:02, 2.11s/it]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0.3333333333333333, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 5%|▌ | 24/450 [00:50<14:22, 2.02s/it]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 6%|▌ | 25/450 [00:51<13:23, 1.89s/it]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 6%|▌ | 26/450 [00:55<17:52, 2.53s/it]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 6%|▌ | 27/450 [00:57<16:05, 2.28s/it]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0.6666666666666666, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 6%|▌ | 28/450 [00:59<15:48, 2.25s/it]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 6%|▋ | 29/450 [01:01<14:51, 2.12s/it]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 7%|▋ | 30/450 [01:03<14:41, 2.10s/it]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 7%|▋ | 31/450 [01:05<13:39, 1.96s/it]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 7%|▋ | 32/450 [01:07<13:29, 1.94s/it]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 1.0, 'em': 1.0, 'acc': 1.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 7%|▋ | 33/450 [01:09<13:31, 1.95s/it]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\r",
"Evaluating workflow: 8%|▊ | 34/450 [01:10<13:09, 1.90s/it]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 10255787.201895358)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 10255792.081686465)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 10255801.009716788)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 10255805.924431246)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 10255812.996734176)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 10255796.767546052)])']\n",
"connector: \n",
"Unclosed connector\n",
"connections: ['deque([(, 10255809.484247368)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 10255821.983979572)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 10255825.766781336)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 10255835.568036309)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 10255842.657826856)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 10255847.257425716)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 10255817.395932108)])']\n",
"connector: \n",
"Unclosed connector\n",
"connections: ['deque([(, 10255830.488791157)])']\n",
"connector: \n",
"Unclosed connector\n",
"connections: ['deque([(, 10255839.443144867)])']\n",
"connector: \n",
"Unclosed connector\n",
"connections: ['deque([(, 10255851.9649534)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 10255856.193006458)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 10255864.05837833)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 10255878.91521714)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 10255885.307229016)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 10255893.775389053)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 10255898.064972663)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 10255860.167464403)])']\n",
"connector: \n",
"Unclosed connector\n",
"connections: ['deque([(, 10255867.450055305)])']\n",
"connector: \n",
"Unclosed connector\n",
"connections: ['deque([(, 10255873.430541812)])']\n",
"connector: \n",
"Unclosed connector\n",
"connections: ['deque([(, 10255903.67291296)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 10255907.024463529)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 10255913.646681244)])']\n",
"connector: \n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 10255921.94653662)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 10255925.31384234)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 10255930.248967316)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 10255910.46948294)])']\n",
"connector: \n",
"Unclosed connector\n",
"connections: ['deque([(, 10255917.445702909)])']\n",
"connector: \n",
"Unclosed connector\n",
"connections: ['deque([(, 10255935.222686874)])']\n",
"connector: \n",
"Unclosed connector\n",
"connections: ['deque([(, 10255940.870964816)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 10255943.893517813)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 10255948.020650452)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 10255963.129046261)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 10255970.70954101)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 10255974.812432844)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 10255953.01871942)])']\n",
"connector: \n",
"Unclosed connector\n",
"connections: ['deque([(, 10255966.924491264)])']\n",
"connector: \n",
"Unclosed connector\n",
"connections: ['deque([(, 10255979.173850534)])']\n",
"connector: \n",
"Unclosed connector\n",
"connections: ['deque([(, 10255982.628446912)])']\n",
"connector: \n",
"Unclosed connector\n",
"connections: ['deque([(, 10255987.43040238)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 10255998.70159754)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 10256012.467695586)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 10256017.08509794)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 10256020.760379571)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 10256008.277802393)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 10255991.08613186)])']\n",
"connector: \n",
"Unclosed connector\n",
"connections: ['deque([(, 10255994.373725573)])']\n",
"connector: \n",
"Unclosed connector\n",
"connections: ['deque([(, 10256002.941956632)])']\n",
"connector: \n",
"Unclosed connector\n",
"connections: ['deque([(, 10256025.355866479)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 10256037.03918762)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 10256041.289793203)])']\n",
"connector: \n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 10256056.52420596)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 10256061.017060949)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 10256032.933675325)])']\n",
"connector: \n",
"Unclosed connector\n",
"connections: ['deque([(, 10256048.780722512)])']\n",
"connector: \n",
"Unclosed connector\n",
"connections: ['deque([(, 10256053.489278905)])']\n",
"connector: \n",
"Unclosed connector\n",
"connections: ['deque([(, 10256065.932307964)])']\n",
"connector: \n",
"Unclosed connector\n",
"connections: ['deque([(, 10256072.339950446)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 10256076.69337796)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 10256081.514825417)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 10256093.625411673)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 10256097.785585972)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 10256106.437148996)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 10256086.58743395)])']\n",
"connector: \n",
"Unclosed connector\n",
"connections: ['deque([(, 10256102.445671204)])']\n",
"connector: \n",
"Unclosed connector\n",
"connections: ['deque([(, 10256110.284031302)])']\n",
"connector: \n",
"Unclosed connector\n",
"connections: ['deque([(, 10256120.055147093)])']\n",
"connector: \n",
"Unclosed connector\n",
"connections: ['deque([(, 10256123.13029033)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 10256126.93858076)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 10256135.417780869)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 10256146.231081365)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 10256149.689610716)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 10256130.870614449)])']\n",
"connector: \n",
"Unclosed connector\n",
"connections: ['deque([(, 10256139.594969466)])']\n",
"connector: \n",
"Unclosed connector\n",
"connections: ['deque([(, 10256143.0568816)])']\n",
"connector: \n",
"Unclosed connector\n",
"connections: ['deque([(, 10256154.405880293)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 10256163.15076304)])']\n",
"connector: \n",
"Unclosed client session\n",
"client_session: \n",
"Unclosed connector\n",
"connections: ['deque([(, 10256171.261501232)])']\n",
"connector: