{ "cells": [ { "cell_type": "code", "execution_count": 17, "id": "4c2a6fa7", "metadata": {}, "outputs": [], "source": [ "import os\n", "\n", "from dotenv import load_dotenv\n", "\n", "from evoagentx.agents.agent_manager import AgentManager\n", "from evoagentx.benchmark import HotPotQA\n", "from evoagentx.core.callbacks import suppress_logger_info\n", "from evoagentx.core.logging import logger\n", "from evoagentx.evaluators import Evaluator\n", "from evoagentx.models import OpenAILLM, OpenAILLMConfig\n", "from evoagentx.optimizers import TextGradOptimizer\n", "from evoagentx.prompts import StringTemplate\n", "from evoagentx.workflow import SequentialWorkFlowGraph\n", "from dotenv import load_dotenv\n", "\n", "from evoagentx.agents.agent_manager import AgentManager\n", "from evoagentx.benchmark import MBPP\n", "from evoagentx.core.callbacks import suppress_logger_info\n", "from evoagentx.core.logging import logger\n", "from evoagentx.evaluators import Evaluator\n", "from evoagentx.models import OpenAILLM, OpenAILLMConfig\n", "from evoagentx.optimizers import TextGradOptimizer\n", "from evoagentx.prompts import StringTemplate\n", "from evoagentx.workflow import SequentialWorkFlowGraph\n", "\n", "from evoagentx.models import OpenAILLMConfig, OpenAILLM\n", "from evoagentx.workflow import SEWWorkFlowGraph, STRUCTUREWorkFlowGraph\n", "from evoagentx.agents import AgentManager\n", "from evoagentx.benchmark import HumanEval,AFlowMBPP\n", "from evoagentx.evaluators import Evaluator \n", "from evoagentx.optimizers import SEWOptimizer, STRUCTUREOptimizer\n", "from evoagentx.optimizers.structure_optimizer import STRUCTUREWorkFlowScheme\n", "from evoagentx.core.callbacks import suppress_logger_info\n", "\n", "from evoagentx.models import OpenAILLMConfig, OpenAILLM,AzureOpenAIConfig,LiteLLMConfig,LiteLLM\n", "from evoagentx.workflow import SEWWorkFlowGraph \n", "from evoagentx.agents import AgentManager\n", "from evoagentx.benchmark import MBPPPLUS, AFlowMBPPPLUS\n", "from evoagentx.evaluators import Evaluator \n", "from evoagentx.optimizers import SEWOptimizer \n", "from evoagentx.core.callbacks import suppress_logger_info\n", "from evoagentx.benchmark import HumanEvalPLUS\n", "from evoagentx.benchmark import SciCode\n", "from evoagentx.benchmark import PubMedQA\n", "from copy import deepcopy\n", "\n", "import nest_asyncio\n", "nest_asyncio.apply()\n", "\n", "class PubMedQASplits(PubMedQA):\n", "\n", " def _load_data(self):\n", " # load the original test data \n", " super()._load_data()\n", " # split the data into train, dev and test\n", " import numpy as np \n", " np.random.seed(42)\n", " permutation = np.random.permutation(len(self._dev_data))\n", " full_test_data = self._dev_data \n", " # randomly select 10 samples for train, 40 for dev, and 100 for test\n", " self._train_data = [full_test_data[idx] for idx in permutation[:50]]\n", " self._dev_data = [full_test_data[idx] for idx in permutation[:50]]\n", " self._test_data =self._test_data[0:500]\n", " self._fulldata = full_test_data\n", "\n", "\n", "def collate_func(example: dict) -> dict:\n", " context_list = []\n", " paragraphs = example[\"context\"][\"contexts\"]\n", " context = \"\\n\".join(paragraphs)\n", " problem = \"Context: {}\\n\\nQuestion: {}\\n\\nAnswer:\".format(context, example[\"question\"])\n", " return {\"problem\": problem}\n", "\n", "\n", "hotpotqa_graph_data = {\n", " \"goal\": \"Answer the question based on the context. The answer should be a direct response to the question, without including explanations or reasoning.\",\n", " \"tasks\": [\n", " {\n", " \"name\": \"answer_generate\",\n", " \"description\": \"Answer the question based on the context.\",\n", " \"inputs\": [\n", " {\"name\": \"problem\", \"type\": \"str\", \"required\": True, \"description\": \"The problem to solve.\"}\n", " ],\n", " \"outputs\": [\n", " {\"name\": \"answer\", \"type\": \"str\", \"required\": True, \"description\": \"The answer to the problem.\"}\n", " ],\n", " \"prompt_template\": StringTemplate(instruction=\"Think step by step to answer the question. You should explain your thinking process in the 'thought' field, and provide the final answer in the 'answer' field.\\nFormat your output in xml format, such as xxx and xxx.\"),\n", " \"parse_mode\": \"xml\"\n", " }\n", " ] \n", "}\n", "\n", "os.environ[\"AZURE_OPENAI_DEPLOYMENT_NAME\"] = \"gpt-4o-mini\"\n", "os.environ[\"AZURE_OPENAI_ENDPOINT\"] = \"https://tianyuliu-hua-raredisea-resource.cognitiveservices.azure.com/\"\n", "os.environ[\"AZURE_OPENAI_KEY\"] = \"2pa9h2ZIN1lQepFWwYADlXIKIansa9KPhxMoumeGbRQ08f2uDTXiJQQJ99BKACHYHv6XJ3w3AAAAACOGsQIt\"\n", "os.environ[\"AZURE_OPENAI_API_VERSION\"] = \"2025-01-01-preview\"\n", "llm_config = LiteLLMConfig(model=\"azure/\" + os.getenv(\"AZURE_OPENAI_DEPLOYMENT_NAME\"), # Azure model format\n", " azure_endpoint=os.getenv(\"AZURE_OPENAI_ENDPOINT\"),\n", " azure_key=os.getenv(\"AZURE_OPENAI_KEY\"),\n", " api_version=os.getenv(\"AZURE_OPENAI_API_VERSION\", \"2024-12-01-preview\"), top_p=0.85, temperature=0.2, frequency_penalty=0.0, presence_penalty=0.0)\n", "\n", "executor_llm = LiteLLM(config=llm_config)\n", "optimizer_llm = LiteLLM(config=llm_config)" ] }, { "cell_type": "code", "execution_count": 18, "id": "ad0efa03", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "evoagentx.optimizers.sew_optimizer.SEWOptimizer" ] }, "execution_count": 18, "metadata": {}, "output_type": "execute_result" } ], "source": [ "SEWOptimizer " ] }, { "cell_type": "code", "execution_count": 19, "id": "ad4b2024", "metadata": {}, "outputs": [], "source": [ "# difficult easy " ] }, { "cell_type": "code", "execution_count": 20, "id": "c95059f0", "metadata": {}, "outputs": [], "source": [ "from evoagentx.benchmark import HotPotQA" ] }, { "cell_type": "code", "execution_count": 21, "id": "84efabfa", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "\u001b[32m2025-12-21 11:19:28.844\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.benchmark.pubmedqa\u001b[0m:\u001b[36m_load_data_from_file\u001b[0m:\u001b[36m51\u001b[0m - \u001b[1mloading HotPotQA data from /gpfs/radev/pi/ying_rex/tl688/selfevolve/EvoAgentX/examples/pubmedqa/pubmedqa_train.json ...\u001b[0m\n", "\u001b[32m2025-12-21 11:19:30.791\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.benchmark.pubmedqa\u001b[0m:\u001b[36m_load_data_from_file\u001b[0m:\u001b[36m51\u001b[0m - \u001b[1mloading HotPotQA data from /gpfs/radev/pi/ying_rex/tl688/selfevolve/EvoAgentX/examples/pubmedqa/pubmedqa_train.json ...\u001b[0m\n", "\u001b[32m2025-12-21 11:19:33.925\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.benchmark.pubmedqa\u001b[0m:\u001b[36m_load_data_from_file\u001b[0m:\u001b[36m51\u001b[0m - \u001b[1mloading HotPotQA data from /gpfs/radev/pi/ying_rex/tl688/selfevolve/EvoAgentX/examples/pubmedqa/pubmedqa_label.json ...\u001b[0m\n" ] } ], "source": [ "# llm_config = OpenAILLMConfig(model=\"gpt-4o-mini-2024-07-18\", openai_key=OPENAI_API_KEY, top_p=0.85, temperature=0.2, frequency_penalty=0.0, presence_penalty=0.0)\n", "# llm = OpenAILLM(config=llm_config)\n", "llm = executor_llm\n", "\n", "# obtain SEW workflow \n", "sew_graph = SEWWorkFlowGraph.from_dict(hotpotqa_graph_data)\n", "agent_manager = AgentManager()\n", "agent_manager.add_agents_from_workflow(sew_graph, executor_llm.config)\n", "\n", "benchmark = PubMedQASplits()\n", "\n", "# obtain Evaluator\n", "evaluator = Evaluator(llm=llm, agent_manager=agent_manager, collate_func=collate_func, num_workers=20, verbose=True)" ] }, { "cell_type": "code", "execution_count": 22, "id": "d2bba683", "metadata": {}, "outputs": [], "source": [ "# import json\n", "# # with open(\"../../MaAS/maas/ext/maas/data/humaneval_train.jsonl\", 'w') as f:\n", "# # json.dump(humaneval._dev_data, f, indent=2) # indent=4 makes the JSON output more readable\n", "\n", "\n", "# # with open(\"../../MaAS/maas/ext/maas/data/humaneval_test.jsonl\", 'w') as f:\n", "# # json.dump(humaneval._test_data, f, indent=2) # indent=4 makes the JSON output more readable\n", "\n", "# with open(\"../../MaAS/maas/ext/maas/data/humaneval_train.jsonl\", 'w') as f:\n", "# for obj in humaneval._dev_data:\n", "# json_line = json.dumps(obj)\n", "# f.write(json_line + '\\n')\n", " \n", "# with open(\"../../MaAS/maas/ext/maas/data/humaneval_test.jsonl\", 'w') as f:\n", "# for obj in humaneval._test_data:\n", "# json_line = json.dumps(obj)\n", "# f.write(json_line + '\\n')\n" ] }, { "cell_type": "code", "execution_count": 23, "id": "8598151b", "metadata": { "scrolled": false }, "outputs": [ { "data": { "text/plain": [ "1" ] }, "execution_count": 23, "metadata": {}, "output_type": "execute_result" } ], "source": [ "len(sew_graph.to_dict()['nodes'])" ] }, { "cell_type": "code", "execution_count": 24, "id": "b1f7fc18", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "0" ] }, "execution_count": 24, "metadata": {}, "output_type": "execute_result" } ], "source": [ "len(sew_graph.edges)" ] }, { "cell_type": "code", "execution_count": 25, "id": "33859fa8", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "[]" ] }, "execution_count": 25, "metadata": {}, "output_type": "execute_result" } ], "source": [ "sew_graph.edges" ] }, { "cell_type": "code", "execution_count": null, "id": "0a464c5e", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": 26, "id": "3c048529", "metadata": { "scrolled": true }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 0%| | 1/500 [00:02<22:32, 2.71s/it]Task exception was never retrieved\n", "future: exception=RuntimeError('Event loop is closed')>\n", "Traceback (most recent call last):\n", " File \"/gpfs/radev/home/tl688/.conda/envs/evoagentx/lib/python3.11/asyncio/tasks.py\", line 277, in __step\n", " result = coro.send(None)\n", " ^^^^^^^^^^^^^^^\n", " File \"/gpfs/radev/home/tl688/.conda/envs/evoagentx/lib/python3.11/site-packages/litellm/utils.py\", line 873, in _client_async_logging_helper\n", " GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(\n", " File \"/gpfs/radev/home/tl688/.conda/envs/evoagentx/lib/python3.11/site-packages/litellm/litellm_core_utils/logging_worker.py\", line 322, in ensure_initialized_and_enqueue\n", " self.enqueue(async_coroutine)\n", " File \"/gpfs/radev/home/tl688/.conda/envs/evoagentx/lib/python3.11/site-packages/litellm/litellm_core_utils/logging_worker.py\", line 131, in enqueue\n", " self._queue.put_nowait(task)\n", " File \"/gpfs/radev/home/tl688/.conda/envs/evoagentx/lib/python3.11/asyncio/queues.py\", line 147, in put_nowait\n", " self._wakeup_next(self._getters)\n", " File \"/gpfs/radev/home/tl688/.conda/envs/evoagentx/lib/python3.11/asyncio/queues.py\", line 63, in _wakeup_next\n", " waiter.set_result(None)\n", " File \"/gpfs/radev/home/tl688/.conda/envs/evoagentx/lib/python3.11/asyncio/futures.py\", line 263, in set_result\n", " self.__schedule_callbacks()\n", " File \"/gpfs/radev/home/tl688/.conda/envs/evoagentx/lib/python3.11/asyncio/futures.py\", line 173, in __schedule_callbacks\n", " self._loop.call_soon(callback, self, context=ctx)\n", " File \"/gpfs/radev/home/tl688/.conda/envs/evoagentx/lib/python3.11/asyncio/base_events.py\", line 762, in call_soon\n", " self._check_closed()\n", " File \"/gpfs/radev/home/tl688/.conda/envs/evoagentx/lib/python3.11/asyncio/base_events.py\", line 520, in _check_closed\n", " raise RuntimeError('Event loop is closed')\n", "RuntimeError: Event loop is closed\n", "Evaluating workflow: 1%| | 4/500 [00:02<04:30, 1.84it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 1%|▏ | 7/500 [00:03<03:02, 2.70it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 2%|▏ | 8/500 [00:03<02:34, 3.19it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 2%|▏ | 9/500 [00:03<02:28, 3.31it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 2%|▏ | 11/500 [00:05<03:42, 2.20it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 3%|▎ | 13/500 [00:05<02:51, 2.84it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 3%|▎ | 14/500 [00:06<02:55, 2.77it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 3%|▎ | 16/500 [00:06<02:09, 3.75it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 3%|▎ | 17/500 [00:07<02:55, 2.75it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 4%|▍ | 22/500 [00:07<01:20, 5.92it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 5%|▍ | 24/500 [00:07<01:05, 7.31it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 5%|▌ | 27/500 [00:07<00:47, 9.91it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 6%|▌ | 31/500 [00:08<00:54, 8.66it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 7%|▋ | 33/500 [00:08<00:50, 9.32it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 7%|▋ | 35/500 [00:09<01:33, 4.98it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 7%|▋ | 36/500 [00:09<01:47, 4.32it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 8%|▊ | 39/500 [00:10<01:27, 5.27it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 8%|▊ | 41/500 [00:10<01:13, 6.26it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 9%|▊ | 43/500 [00:10<01:15, 6.09it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 9%|▉ | 44/500 [00:11<01:15, 6.00it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 9%|▉ | 46/500 [00:11<01:10, 6.44it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 10%|▉ | 48/500 [00:11<01:23, 5.41it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 10%|█ | 50/500 [00:12<01:18, 5.76it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 11%|█ | 54/500 [00:12<01:07, 6.57it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 11%|█ | 55/500 [00:13<01:22, 5.40it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 11%|█▏ | 57/500 [00:13<01:09, 6.41it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 12%|█▏ | 60/500 [00:13<00:54, 8.12it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 12%|█▏ | 62/500 [00:14<01:25, 5.14it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 13%|█▎ | 63/500 [00:14<01:23, 5.24it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 13%|█▎ | 65/500 [00:14<01:26, 5.03it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 13%|█▎ | 67/500 [00:14<01:00, 7.11it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 14%|█▍ | 71/500 [00:15<00:50, 8.55it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 15%|█▍ | 73/500 [00:15<01:04, 6.59it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 15%|█▌ | 75/500 [00:16<01:08, 6.18it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 16%|█▌ | 79/500 [00:16<00:57, 7.28it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 16%|█▌ | 80/500 [00:16<01:07, 6.26it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 16%|█▌ | 81/500 [00:17<01:52, 3.73it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 17%|█▋ | 86/500 [00:18<01:01, 6.72it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 17%|█▋ | 87/500 [00:18<01:00, 6.80it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 18%|█▊ | 89/500 [00:18<01:15, 5.43it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 18%|█▊ | 91/500 [00:18<01:10, 5.81it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 18%|█▊ | 92/500 [00:19<01:03, 6.38it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 19%|█▊ | 93/500 [00:19<01:10, 5.79it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 19%|█▉ | 97/500 [00:19<00:48, 8.29it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 20%|█▉ | 99/500 [00:20<01:00, 6.61it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 20%|██ | 100/500 [00:20<00:57, 6.91it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 21%|██ | 104/500 [00:21<01:05, 6.03it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 21%|██ | 106/500 [00:21<01:04, 6.13it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 21%|██▏ | 107/500 [00:21<01:19, 4.95it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 22%|██▏ | 109/500 [00:22<01:18, 4.96it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 23%|██▎ | 114/500 [00:22<00:44, 8.61it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 23%|██▎ | 117/500 [00:22<00:36, 10.37it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 24%|██▍ | 119/500 [00:23<01:04, 5.89it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 24%|██▍ | 122/500 [00:23<01:01, 6.14it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 25%|██▍ | 123/500 [00:24<01:02, 6.02it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 25%|██▌ | 125/500 [00:24<01:06, 5.66it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 25%|██▌ | 127/500 [00:24<00:53, 6.98it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 26%|██▌ | 130/500 [00:25<00:48, 7.70it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 26%|██▌ | 131/500 [00:25<00:52, 6.98it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 27%|██▋ | 136/500 [00:25<00:43, 8.39it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 27%|██▋ | 137/500 [00:26<01:07, 5.35it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 28%|██▊ | 138/500 [00:26<01:14, 4.85it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 28%|██▊ | 141/500 [00:26<00:59, 5.99it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 29%|██▉ | 145/500 [00:27<00:50, 7.08it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 30%|██▉ | 148/500 [00:27<00:47, 7.40it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 30%|███ | 152/500 [00:28<00:47, 7.31it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 31%|███ | 153/500 [00:28<01:11, 4.85it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 31%|███ | 155/500 [00:29<01:05, 5.24it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 31%|███▏ | 157/500 [00:29<01:07, 5.08it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 32%|███▏ | 158/500 [00:29<01:09, 4.94it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 32%|███▏ | 160/500 [00:30<01:03, 5.32it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 32%|███▏ | 162/500 [00:30<01:04, 5.26it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 33%|███▎ | 165/500 [00:30<00:44, 7.51it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 34%|███▎ | 168/500 [00:31<00:45, 7.35it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 34%|███▍ | 169/500 [00:31<00:56, 5.84it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 34%|███▍ | 172/500 [00:31<00:44, 7.30it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 35%|███▍ | 174/500 [00:32<00:38, 8.45it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 35%|███▌ | 175/500 [00:32<00:55, 5.89it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 35%|███▌ | 177/500 [00:32<01:00, 5.31it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 36%|███▌ | 179/500 [00:33<00:51, 6.25it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 36%|███▋ | 182/500 [00:33<00:46, 6.89it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 37%|███▋ | 183/500 [01:00<25:13, 4.78s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 37%|███▋ | 187/500 [01:01<12:27, 2.39s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 38%|███▊ | 192/500 [01:02<05:38, 1.10s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 39%|███▊ | 193/500 [01:03<05:43, 1.12s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "\u001b[32m2025-12-21 11:20:50.300\u001b[0m | \u001b[31m\u001b[1mERROR \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 39%|███▉ | 194/500 [01:03<04:43, 1.08it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "\u001b[32m2025-12-21 11:20:50.591\u001b[0m | \u001b[31m\u001b[1mERROR \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 39%|███▉ | 195/500 [01:03<03:59, 1.27it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "\u001b[32m2025-12-21 11:20:50.607\u001b[0m | \u001b[31m\u001b[1mERROR \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n", "\u001b[32m2025-12-21 11:20:50.613\u001b[0m | \u001b[31m\u001b[1mERROR \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 40%|███▉ | 198/500 [01:03<02:07, 2.37it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 40%|████ | 201/500 [01:04<01:36, 3.11it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 40%|████ | 202/500 [01:05<02:17, 2.16it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 41%|████ | 206/500 [01:06<01:21, 3.60it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 41%|████▏ | 207/500 [01:06<01:13, 3.99it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 42%|████▏ | 208/500 [01:07<02:16, 2.13it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 43%|████▎ | 214/500 [01:08<00:58, 4.89it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 43%|████▎ | 216/500 [01:08<00:59, 4.80it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 44%|████▎ | 218/500 [01:08<00:48, 5.76it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 44%|████▍ | 219/500 [01:09<01:00, 4.68it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 45%|████▍ | 224/500 [01:09<00:40, 6.88it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 45%|████▌ | 227/500 [01:09<00:36, 7.49it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 46%|████▌ | 228/500 [01:10<00:44, 6.11it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 46%|████▌ | 231/500 [01:10<00:53, 5.01it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 47%|████▋ | 235/500 [01:11<00:33, 7.93it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 47%|████▋ | 237/500 [01:11<00:34, 7.72it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 48%|████▊ | 240/500 [01:12<00:35, 7.37it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 48%|████▊ | 242/500 [01:12<00:27, 9.26it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 49%|████▉ | 244/500 [01:12<00:27, 9.34it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 49%|████▉ | 246/500 [01:13<00:47, 5.34it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 49%|████▉ | 247/500 [01:13<00:50, 5.05it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 50%|████▉ | 249/500 [01:13<00:49, 5.03it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 50%|█████ | 252/500 [01:13<00:30, 8.03it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 51%|█████ | 254/500 [01:14<00:39, 6.18it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 51%|█████▏ | 257/500 [01:15<00:50, 4.82it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 52%|█████▏ | 259/500 [01:15<00:42, 5.70it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 52%|█████▏ | 261/500 [01:15<00:41, 5.80it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 53%|█████▎ | 263/500 [01:16<00:39, 5.93it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 53%|█████▎ | 266/500 [01:16<00:38, 6.14it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 53%|█████▎ | 267/500 [01:16<00:37, 6.29it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 54%|█████▍ | 269/500 [01:17<00:38, 6.06it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 54%|█████▍ | 270/500 [01:17<00:46, 4.99it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 55%|█████▍ | 273/500 [01:17<00:35, 6.42it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 55%|█████▌ | 276/500 [01:18<00:32, 6.82it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 56%|█████▌ | 280/500 [01:18<00:21, 10.35it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 57%|█████▋ | 284/500 [01:19<00:32, 6.60it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 57%|█████▋ | 286/500 [01:19<00:36, 5.79it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 57%|█████▋ | 287/500 [01:19<00:34, 6.26it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 58%|█████▊ | 288/500 [01:20<00:39, 5.33it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 58%|█████▊ | 291/500 [01:20<00:33, 6.24it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 59%|█████▉ | 294/500 [01:20<00:26, 7.91it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 59%|█████▉ | 296/500 [01:21<00:29, 6.86it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 60%|█████▉ | 298/500 [01:21<00:24, 8.32it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 60%|█████▉ | 299/500 [01:21<00:43, 4.58it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 60%|██████ | 302/500 [01:22<00:35, 5.52it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 61%|██████▏ | 307/500 [01:22<00:20, 9.44it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 62%|██████▏ | 309/500 [01:23<00:23, 8.18it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 63%|██████▎ | 313/500 [01:24<00:31, 6.00it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 63%|██████▎ | 314/500 [01:24<00:30, 6.16it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 63%|██████▎ | 317/500 [01:24<00:31, 5.80it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 64%|██████▍ | 320/500 [01:25<00:25, 7.03it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 65%|██████▍ | 323/500 [01:25<00:17, 9.90it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 65%|██████▌ | 325/500 [01:25<00:21, 8.23it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 66%|██████▌ | 329/500 [01:25<00:15, 11.05it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 66%|██████▌ | 331/500 [01:26<00:29, 5.65it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 67%|██████▋ | 333/500 [01:27<00:32, 5.07it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 67%|██████▋ | 335/500 [01:27<00:30, 5.47it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 68%|██████▊ | 340/500 [01:28<00:24, 6.41it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 68%|██████▊ | 342/500 [01:28<00:29, 5.41it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 69%|██████▉ | 346/500 [01:29<00:21, 7.33it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 70%|██████▉ | 348/500 [01:29<00:20, 7.49it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 70%|███████ | 350/500 [01:29<00:28, 5.28it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 70%|███████ | 352/500 [01:30<00:29, 5.05it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 71%|███████ | 354/500 [01:30<00:25, 5.82it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 71%|███████▏ | 357/500 [01:31<00:21, 6.70it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 72%|███████▏ | 359/500 [01:31<00:25, 5.54it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 72%|███████▏ | 362/500 [01:31<00:20, 6.89it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 73%|███████▎ | 364/500 [01:32<00:17, 7.66it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 73%|███████▎ | 366/500 [01:32<00:14, 9.01it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 74%|███████▎ | 368/500 [01:32<00:15, 8.62it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 74%|███████▍ | 370/500 [01:32<00:17, 7.54it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 74%|███████▍ | 372/500 [01:33<00:21, 6.05it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 75%|███████▍ | 373/500 [02:01<12:43, 6.01s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 75%|███████▍ | 374/500 [02:01<09:51, 4.69s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "\u001b[32m2025-12-21 11:21:48.414\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n", "Example: {'pubid': 21346501, 'question': \"Can students' scores on preclerkship clinical performance examinations predict that they will fail a senior clinical performance examination?\", 'context': {'contexts': ['This study was designed to determine whether preclerkship performance examinations could accurately identify medical students at risk for failing a senior clinical performance examination (CPE).', \"This study used a retrospective case-control, multiyear design, with contingency table analyses, to examine the performance of 412 students in the classes of 2005 to 2010 at a midwestern medical school. During their second year, these students took four CPEs that each used three standardized patient (SP) cases, for a total of 12 cases. The authors correlated each student's average year 2 case score with the student's average case score on a senior (year 4) CPE. Contingency table analysis was carried out using performance on the year 2 CPEs and passing/failing the senior CPE. Similar analyses using each student's United States Medical Licensing Examination (USMLE) Step 1 scores were also performed. Sensitivity, specificity, odds ratio, and relative risk were calculated for two year 2 performance standards.\", \"Students' low performances relative to their class on the year 2 CPEs were a strong predictor that they would fail the senior CPE. Their USMLE Step 1 scores also correlated with their performance on the senior CPE, although the predictive values for these scores were considerably weaker.\"], 'labels': ['PURPOSE', 'METHOD', 'RESULTS'], 'meshes': ['Case-Control Studies', 'Chi-Square Distribution', 'Clinical Clerkship', 'Clinical Competence', 'Education, Medical, Undergraduate', 'Educational Measurement', 'Educational Status', 'Humans', 'Illinois', 'Licensure', 'Patient Simulation', 'Predictive Value of Tests', 'Retrospective Studies', 'Risk Factors', 'Sensitivity and Specificity', 'United States'], 'reasoning_required_pred': ['y', 'e', 's'], 'reasoning_free_pred': ['y', 'e', 's']}, 'long_answer': 'Under the conditions of this study, preclerkship (year 2) CPEs strongly predicted medical students at risk for failing a senior CPE. This finding opens the opportunity for remediation of deficits prior to or during clerkships.', 'final_decision': 'yes'}\n", "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 2 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 76%|███████▌ | 379/500 [02:01<03:08, 1.56s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 76%|███████▌ | 381/500 [02:01<02:11, 1.10s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "\u001b[32m2025-12-21 11:21:48.865\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n", "Example: {'pubid': 27690714, 'question': 'Can EMS Providers Provide Appropriate Tidal Volumes in a Simulated Adult-sized Patient with a Pediatric-sized Bag-Valve-Mask?', 'context': {'contexts': ['In the prehospital setting, Emergency Medical Services (EMS) professionals rely on providing positive pressure ventilation with a bag-valve-mask (BVM). Multiple emergency medicine and critical care studies have shown that lung-protective ventilation protocols reduce morbidity and mortality. Our primary objective was to determine if a group of EMS professionals could provide ventilations with a smaller BVM that would be sufficient to ventilate patients. Secondary objectives included 1) if the pediatric bag provided volumes similar to lung-protective ventilation in the hospital setting and 2) compare volumes provided to the patient depending on the type of airway (mask, King tube, and intubation).', 'Using a patient simulator of a head and thorax that was able to record respiratory rate, tidal volume, peak pressure, and minute volume via a laptop computer, participants were asked to ventilate the simulator during six 1-minute ventilation tests. The first scenario was BVM ventilation with an oropharyngeal airway in place ventilating with both an adult- and pediatric-sized BVM, the second scenario had a supraglottic airway and both bags, and the third scenario had an endotracheal tube and both bags. Participants were enrolled in convenience manner while they were on-duty and the research staff was able to travel to their stations. Prior to enrolling, participants were not given any additional training on ventilation skills.', 'We enrolled 50 providers from a large, busy, urban fire-based EMS agency with 14.96 (SD = 9.92) mean years of experience. Only 1.5% of all breaths delivered with the pediatric BVM during the ventilation scenarios were below the recommended tidal volume. A greater percentage of breaths delivered in the recommended range occurred when the pediatric BVM was used (17.5% vs 5.1%, p<0.001). Median volumes for each scenario were 570.5mL, 664.0mL, 663.0mL for the pediatric BMV and 796.0mL, 994.5mL, 981.5mL for the adult BVM. In all three categories of airway devices, the pediatric BVM provided lower median tidal volumes (p<0.001).'], 'labels': ['INTRODUCTION', 'METHODS', 'RESULTS'], 'meshes': ['Adult', 'Emergency Medical Services', 'Female', 'Humans', 'Male', 'Manikins', 'Middle Aged', 'Respiration, Artificial', 'Respiratory Insufficiency', 'Resuscitation', 'Tidal Volume'], 'reasoning_required_pred': ['y', 'e', 's'], 'reasoning_free_pred': ['y', 'e', 's']}, 'long_answer': 'The study suggests that ventilating an adult patient is possible with a smaller, pediatric-sized BVM. The tidal volumes recorded with the pediatric BVM were more consistent with lung-protective ventilation volumes.', 'final_decision': 'yes'}\n", "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 77%|███████▋ | 384/500 [02:01<01:19, 1.45it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m2025-12-21 11:21:50.723\u001b[0m | \u001b[31m\u001b[1mERROR \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 77%|███████▋ | 386/500 [02:03<01:26, 1.32it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "\u001b[32m2025-12-21 11:21:50.794\u001b[0m | \u001b[31m\u001b[1mERROR \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "\u001b[32m2025-12-21 11:21:50.853\u001b[0m | \u001b[31m\u001b[1mERROR \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "\u001b[32m2025-12-21 11:21:50.866\u001b[0m | \u001b[31m\u001b[1mERROR \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 78%|███████▊ | 392/500 [02:03<00:39, 2.77it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "\u001b[32m2025-12-21 11:21:51.694\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n", "Example: {'pubid': 27338535, 'question': 'Do risk calculators accurately predict surgical site\\xa0occurrences?', 'context': {'contexts': ['Current risk assessment models for surgical site occurrence (SSO) and surgical site infection (SSI) after open ventral hernia repair (VHR) have limited external validation. Our aim was to determine (1) whether existing models stratify patients into groups by risk and (2) which model best predicts the rate of SSO and SSI.', \"Patients who underwent open VHR and were followed for at least 1\\xa0mo were included. Using two data sets-a retrospective multicenter database (Ventral Hernia Outcomes Collaborative) and a single-center prospective database (Prospective)-each patient was assigned a predicted risk with each of the following models: Ventral Hernia Risk Score (VHRS), Ventral Hernia Working Group (VHWG), Centers for Disease Control and Prevention Wound Class, and Hernia Wound Risk Assessment Tool (HW-RAT). Patients in the Prospective database were also assigned a predicted risk from the American College of Surgeons National Surgical Quality Improvement Program (ACS-NSQIP). Areas under the receiver operating characteristic curve (area under the curve [AUC]) were compared to assess the predictive accuracy of the models for SSO and SSI. Pearson's chi-square was used to determine which models were able to risk-stratify patients into groups with significantly differing rates of actual SSO and SSI.\", 'The Ventral Hernia Outcomes Collaborative database (n\\xa0=\\xa0795) had an overall SSO and SSI rate of 23% and 17%, respectively. The AUCs were low for SSO (0.56, 0.54, 0.52, and 0.60) and SSI (0.55, 0.53, 0.50, and 0.58). The VHRS (P\\xa0=\\xa00.01) and HW-RAT (P\\xa0<\\xa00.01) significantly stratified patients into tiers for SSO, whereas the VHWG (P\\xa0<\\xa00.05) and HW-RAT (P\\xa0<\\xa00.05) stratified for SSI. In the Prospective database (n\\xa0=\\xa088), 14% and 8% developed an SSO and SSI, respectively. The AUCs were low for SSO (0.63, 0.54, 0.50, 0.57, and 0.69) and modest for SSI (0.81, 0.64, 0.55, 0.62, and 0.73). The ACS-NSQIP (P\\xa0<\\xa00.01) stratified for SSO, whereas the VHRS (P\\xa0<\\xa00.01) and ACS-NSQIP (P\\xa0<\\xa00.05) stratified for SSI. In both databases, VHRS, VHWG, and Centers for Disease Control and Prevention overestimated risk of SSO and SSI, whereas HW-RAT and ACS-NSQIP underestimated risk for all groups.'], 'labels': ['INTRODUCTION', 'METHODS', 'RESULTS'], 'meshes': ['Adult', 'Aged', 'Databases, Factual', 'Decision Support Techniques', 'Female', 'Follow-Up Studies', 'Herniorrhaphy', 'Humans', 'Male', 'Middle Aged', 'Prognosis', 'ROC Curve', 'Retrospective Studies', 'Risk Assessment', 'Risk Factors', 'Surgical Wound Infection'], 'reasoning_required_pred': ['n', 'o'], 'reasoning_free_pred': ['n', 'o']}, 'long_answer': 'All five existing predictive models have limited ability to risk-stratify patients and accurately assess risk of SSO. However, both the VHRS and ACS-NSQIP demonstrate modest success in identifying patients at risk for SSI. Continued model refinement is needed to improve the two highest performing models (VHRS and ACS-NSQIP) along with investigation to determine whether modifications to perioperative management based on risk stratification can improve outcomes.', 'final_decision': 'no'}\n", "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 79%|███████▉ | 395/500 [02:04<00:35, 2.99it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m2025-12-21 11:21:51.785\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n", "Example: {'pubid': 16319544, 'question': 'Is high-sensitivity C-reactive protein associated with carotid atherosclerosis in healthy Koreans?', 'context': {'contexts': ['There is a positive association between chronic inflammation and the risk of cardiovascular disease, but whether there is an association between C-reactive protein (CRP) and carotid atherosclerosis is controversial. We investigated the relationship between high-sensitivity CRP (hsCRP) levels and carotid intima-media thickness (IMT) in healthy Koreans.', 'We measured hsCRP levels, the carotid IMT, and conventional cardiovascular risk factors including obesity parameters, blood pressure, lipid profiles, insulin resistance, and smoking habits in 820 volunteers (35-79 years old) in a cross-sectional study.', 'Higher hsCRP quartile groups had higher mean IMTs, as compared with the lowest quartile (P<0.001 for the trend across quartiles). However, after adjustment for age, the relationship between hsCRP level and IMT was substantially weaker (P = 0.018). After additional adjustments for conventional cardiovascular risk factors, no significant association was observed (P = 0.548). The unadjusted risk for a high carotid IMT value (>or = 1.0 mm) was also positively related to hsCRP quartile, but this relationship was not significant after adjustment for age and other cardiovascular risk factors.'], 'labels': ['BACKGROUND', 'DESIGN', 'RESULTS'], 'meshes': ['Adult', 'Aged', 'Biomarkers', 'C-Reactive Protein', 'Carotid Artery Diseases', 'Carotid Artery, Common', 'Cross-Sectional Studies', 'Female', 'Humans', 'Korea', 'Male', 'Middle Aged', 'Nephelometry and Turbidimetry', 'Prevalence', 'Reference Values', 'Retrospective Studies', 'Risk Factors', 'Tunica Intima', 'Ultrasonography'], 'reasoning_required_pred': ['n', 'o'], 'reasoning_free_pred': ['n', 'o']}, 'long_answer': 'Both hsCRP levels and the carotid IMT were strongly correlated with conventional cardiovascular risk factors, but there was no independent association between hsCRP levels and carotid IMT in healthy Korean adults.', 'final_decision': 'no'}\n", "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 79%|███████▉ | 397/500 [02:05<00:39, 2.59it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 80%|███████▉ | 399/500 [02:06<00:34, 2.97it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "\u001b[32m2025-12-21 11:21:53.830\u001b[0m | \u001b[31m\u001b[1mERROR \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 80%|████████ | 402/500 [02:06<00:28, 3.39it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 81%|████████ | 404/500 [02:07<00:24, 3.87it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 81%|████████ | 405/500 [02:07<00:29, 3.25it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 81%|████████ | 406/500 [02:08<00:28, 3.32it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 82%|████████▏ | 409/500 [02:08<00:19, 4.69it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 82%|████████▏ | 412/500 [02:08<00:12, 7.06it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 83%|████████▎ | 416/500 [02:09<00:10, 7.83it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 84%|████████▎ | 418/500 [02:10<00:20, 3.95it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 84%|████████▍ | 420/500 [02:10<00:18, 4.28it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 85%|████████▌ | 425/500 [02:10<00:10, 7.50it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 85%|████████▌ | 427/500 [02:11<00:09, 7.35it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 86%|████████▌ | 429/500 [02:11<00:11, 6.13it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 86%|████████▋ | 432/500 [02:12<00:10, 6.80it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 87%|████████▋ | 434/500 [02:12<00:09, 7.18it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 87%|████████▋ | 436/500 [02:12<00:09, 6.98it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 88%|████████▊ | 439/500 [02:13<00:08, 7.40it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 88%|████████▊ | 442/500 [02:13<00:07, 8.19it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 89%|████████▊ | 443/500 [02:13<00:10, 5.61it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 90%|████████▉ | 448/500 [02:14<00:08, 6.23it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 90%|█████████ | 452/500 [02:14<00:05, 9.40it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 91%|█████████▏| 457/500 [02:16<00:07, 6.00it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 92%|█████████▏| 461/500 [02:16<00:05, 7.38it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 93%|█████████▎| 463/500 [02:16<00:05, 6.62it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 93%|█████████▎| 464/500 [02:17<00:07, 5.11it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 93%|█████████▎| 466/500 [02:17<00:06, 5.25it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 94%|█████████▎| 468/500 [02:18<00:05, 5.83it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 94%|█████████▍| 469/500 [02:18<00:06, 5.02it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 95%|█████████▍| 473/500 [02:19<00:04, 5.89it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 95%|█████████▌| 475/500 [02:19<00:04, 6.03it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 96%|█████████▌| 479/500 [02:19<00:03, 6.69it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 96%|█████████▌| 480/500 [02:20<00:03, 6.18it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 96%|█████████▋| 482/500 [02:20<00:03, 5.43it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 98%|█████████▊| 488/500 [02:21<00:01, 8.42it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 98%|█████████▊| 490/500 [02:21<00:01, 7.59it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 98%|█████████▊| 491/500 [02:21<00:01, 4.85it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 98%|█████████▊| 492/500 [02:22<00:02, 3.95it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 99%|█████████▉| 494/500 [02:22<00:01, 4.54it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 99%|█████████▉| 496/500 [02:22<00:00, 5.31it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 99%|█████████▉| 497/500 [02:23<00:00, 4.24it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 100%|█████████▉| 498/500 [02:23<00:00, 3.53it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 100%|█████████▉| 499/500 [02:24<00:00, 3.53it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 100%|██████████| 500/500 [02:30<00:00, 3.31it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 0%| | 1/500 [00:01<14:01, 1.69s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 0%| | 2/500 [00:01<07:03, 1.18it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 1%| | 5/500 [00:02<02:28, 3.34it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 2%|▏ | 9/500 [00:02<01:30, 5.42it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 3%|▎ | 15/500 [00:03<00:40, 11.87it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 4%|▍ | 19/500 [00:03<00:47, 10.11it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 4%|▍ | 21/500 [00:04<01:30, 5.30it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 5%|▍ | 23/500 [00:04<01:35, 5.01it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 5%|▌ | 25/500 [00:04<01:10, 6.74it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 6%|▌ | 30/500 [00:05<00:45, 10.29it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 7%|▋ | 33/500 [00:05<00:45, 10.24it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 7%|▋ | 36/500 [00:05<00:48, 9.48it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 8%|▊ | 38/500 [00:06<01:20, 5.72it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 8%|▊ | 41/500 [00:07<01:19, 5.75it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 9%|▉ | 44/500 [00:07<00:58, 7.77it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 9%|▉ | 46/500 [00:07<00:51, 8.78it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 10%|█ | 50/500 [00:08<00:47, 9.38it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 11%|█ | 54/500 [00:08<00:44, 10.09it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 11%|█ | 56/500 [00:09<01:20, 5.48it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 11%|█▏ | 57/500 [00:09<01:25, 5.21it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 12%|█▏ | 59/500 [00:09<01:21, 5.41it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 13%|█▎ | 63/500 [00:10<01:08, 6.37it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 14%|█▍ | 69/500 [00:10<00:40, 10.62it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 14%|█▍ | 71/500 [00:11<00:56, 7.64it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 14%|█▍ | 72/500 [00:30<21:35, 3.03s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "\u001b[32m2025-12-21 11:22:48.578\u001b[0m | \u001b[31m\u001b[1mERROR \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 2 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 15%|█▍ | 74/500 [00:30<15:23, 2.17s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "\u001b[32m2025-12-21 11:22:48.585\u001b[0m | \u001b[31m\u001b[1mERROR \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 2 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 15%|█▌ | 77/500 [00:30<09:21, 1.33s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "\u001b[32m2025-12-21 11:22:48.841\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n", "Example: {'pubid': 29112560, 'question': 'Is the Distance Worth It?', 'context': {'contexts': ['It is unclear whether traveling long distances to high-volume centers would compensate for travel burden among patients undergoing rectal cancer resection.', 'The purpose of this study was to determine whether operative volume outweighs the advantages of being treated locally by comparing the outcomes of patients with rectal cancer treated at local, low-volume centers versus far, high-volume centers.', 'This was a population-based study.', 'The National Cancer Database was queried for patients with rectal cancer.', 'Patients with stage II or III rectal cancer who underwent surgical resection between 2006 and 2012 were included.', 'The outcomes of interest were margins, lymph node yield, receipt of neoadjuvant chemoradiation, adjuvant chemotherapy, readmission within 30 days, 30-day and 90-day mortality, and 5-year overall survival.', 'A total of 18,605 patients met inclusion criteria; 2067 patients were in the long-distance/high-volume group and 1362 in the short-distance/low-volume group. The median travel distance was 62.6 miles for the long-distance/high-volume group and 2.3 miles for the short-distance/low-volume group. Patients who were younger, white, privately insured, and stage III were more likely to have traveled to a high-volume center. When controlled for patient factors, stage, and hospital factors, patients in the short-distance/low-volume group had lower odds of a lymph node yield ≥12 (OR = 0.51) and neoadjuvant chemoradiation (OR = 0.67) and higher 30-day (OR = 3.38) and 90-day mortality (OR = 2.07) compared with those in the long-distance/high-volume group. The short-distance/low-volume group had a 34% high risk of overall mortality at 5 years compared with the long-distance/high-volume group.', 'We lacked data regarding patient and physician decision making and surgeon-specific factors.'], 'labels': ['BACKGROUND', 'OBJECTIVE', 'DESIGN', 'SETTINGS', 'PATIENTS', 'MAIN OUTCOME MEASURES', 'RESULTS', 'LIMITATIONS'], 'meshes': ['Adenocarcinoma', 'Adenocarcinoma, Mucinous', 'Aged', 'Chemoradiotherapy', 'Chemotherapy, Adjuvant', 'Female', 'Health Services Accessibility', 'Hospitals, High-Volume', 'Humans', 'Lymph Node Excision', 'Male', 'Margins of Excision', 'Middle Aged', 'Neoplasm Staging', 'Patient Readmission', 'Rectal Neoplasms', 'Risk Factors', 'Survival Rate', 'Travel', 'Treatment Outcome', 'United States'], 'reasoning_required_pred': ['y', 'e', 's'], 'reasoning_free_pred': ['y', 'e', 's']}, 'long_answer': 'Our results indicate that when controlled for patient, tumor, and hospital factors, patients who traveled a long distance to a high-volume center had improved lymph node yield, neoadjuvant chemoradiation receipt, and 30- and 90-day mortality compared with those who traveled a short distance to a low-volume center. They also had improved 5-year survival. See Video Abstract at http://links.lww.com/DCR/A446.', 'final_decision': 'yes'}\n", "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 16%|█▌ | 79/500 [00:30<06:55, 1.01it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "\u001b[32m2025-12-21 11:22:48.870\u001b[0m | \u001b[31m\u001b[1mERROR \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "\u001b[32m2025-12-21 11:22:48.935\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n", "Example: {'pubid': 14518645, 'question': 'Injury and poisoning mortality among young men--are there any common factors amenable to prevention?', 'context': {'contexts': ['Deaths from injury and poisoning (suicide, accidents, undetermined deaths, and homicide) are the major cause of death among young men aged 15-39 years in England and Wales and have been increasing in recent years.AIM: To describe common characteristics among young men who die from injury and poisoning.', \"We employed a retrospective survey methodology to investigate factors associated with deaths by injury and poisoning among young men aged 15-39 years (n = 268) in Merseyside and Cheshire during 1995. Data were collected from Coroner's inquest notes and General Practitioner records.\", 'The most common cause of death was poisoning by alcohol and drugs (29.1%, n = 78). A high proportion of cases were unemployed (39.4%, n = 106). Cases were also more likely to be single compared to the general population (74.2% vs 55.5%). Self-destructive behaviour was evident in 77% of deaths (n = 206).'], 'labels': ['BACKGROUND', 'DESIGN', 'MAIN RESULTS'], 'meshes': ['Adolescent', 'Adult', 'Cause of Death', 'Data Collection', 'England', 'Humans', 'Male', 'Poisoning', 'Retrospective Studies', 'Risk Factors', 'Self-Injurious Behavior', 'Substance-Related Disorders', 'Suicide', 'Wales', 'Wounds and Injuries'], 'reasoning_required_pred': ['y', 'e', 's'], 'reasoning_free_pred': ['y', 'e', 's']}, 'long_answer': 'Alcohol and drug use are important contributory factors to injury and poisoning deaths. More research is needed into the effects of unemployment and being single on the health of young men, and to investigate the motivations behind risk taking and self-destructive behaviour.', 'final_decision': 'yes'}\n", "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 17%|█▋ | 83/500 [00:31<04:05, 1.70it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 17%|█▋ | 85/500 [00:33<04:32, 1.52it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "\u001b[32m2025-12-21 11:22:50.998\u001b[0m | \u001b[31m\u001b[1mERROR \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "\u001b[32m2025-12-21 11:22:51.010\u001b[0m | \u001b[31m\u001b[1mERROR \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 17%|█▋ | 87/500 [00:33<03:28, 1.98it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "\u001b[32m2025-12-21 11:22:51.066\u001b[0m | \u001b[31m\u001b[1mERROR \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 18%|█▊ | 89/500 [00:33<02:41, 2.54it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "\u001b[32m2025-12-21 11:22:51.280\u001b[0m | \u001b[31m\u001b[1mERROR \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 18%|█▊ | 92/500 [00:33<02:01, 3.36it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "\u001b[32m2025-12-21 11:22:52.391\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n", "Example: {'pubid': 17562682, 'question': 'Is Panton-Valentine leucocidin associated with the pathogenesis of Staphylococcus aureus bacteraemia in the UK?', 'context': {'contexts': ['The morbidity and mortality associated with Panton-Valentine leucocidin (PVL)-positive Staphylococcus aureus suggest that this toxin is a key marker of disease severity. Nevertheless, the importance of PVL in the pathogenesis of primary bacteraemia caused by S. aureus is uncertain. We have determined the prevalence of PVL-encoding genes among isolates of S. aureus from bacteraemic patients.', 'Consecutive bacteraemia isolates of S. aureus (n=244) from patients hospitalized in 25 centres in the UK and Ireland during 2005 were screened for PVL and mecA genes. PVL-positive isolates were characterized by toxin gene profiling, PFGE, spa-typing and MIC determinations for a range of antimicrobials.', 'Four out of 244 isolates (1.6%) were PVL-positive and susceptible to oxacillin [methicillin-susceptible S. aureus (MSSA)]. Eighty-eight out of 244 (36%) were oxacillin-resistant (methicillin-resistant S. aureus), but none was PVL-positive. The four patients (two males: 30 and 33 years; two females: 62 and 80 years) had infection foci of: skin and soft tissue, unknown, indwelling line, and surgical site, and were located at one centre in Wales, one in England and two in Ireland. One of four PVL-positive isolates was resistant to penicillin and fusidic acid, the remainder were susceptible to all antibiotics tested. Genotypic analyses showed that the four isolates represented three distinct strains; the two isolates from Ireland were related.'], 'labels': ['BACKGROUND', 'METHODS', 'RESULTS'], 'meshes': ['Adolescent', 'Adult', 'Aged', 'Aged, 80 and over', 'Anti-Bacterial Agents', 'Bacteremia', 'Bacterial Proteins', 'Bacterial Toxins', 'Child', 'Child, Preschool', 'Exotoxins', 'Female', 'Fusidic Acid', 'Humans', 'Infant', 'Ireland', 'Leukocidins', 'Male', 'Methicillin Resistance', 'Microbial Sensitivity Tests', 'Middle Aged', 'Oxacillin', 'Penicillin Resistance', 'Penicillin-Binding Proteins', 'Staphylococcal Infections', 'Staphylococcus aureus', 'United Kingdom'], 'reasoning_required_pred': ['m', 'a', 'y', 'b', 'e'], 'reasoning_free_pred': ['n', 'o']}, 'long_answer': 'We found that 1.6% of S. aureus (all MSSA) from bacteraemic patients were PVL-positive. This low incidence suggests that PVL-positive S. aureus are of no particular significance as causative agents of S. aureus bacteraemia.', 'final_decision': 'maybe'}\n", "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 19%|█▉ | 94/500 [00:34<02:22, 2.85it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 19%|█▉ | 97/500 [00:35<01:54, 3.51it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 20%|█▉ | 99/500 [00:35<01:27, 4.60it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "\u001b[32m2025-12-21 11:22:53.720\u001b[0m | \u001b[31m\u001b[1mERROR \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 20%|██ | 101/500 [00:35<01:20, 4.98it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 21%|██ | 105/500 [00:36<01:11, 5.50it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 21%|██ | 106/500 [00:37<01:41, 3.90it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 21%|██▏ | 107/500 [00:37<01:45, 3.74it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 22%|██▏ | 109/500 [00:37<01:22, 4.75it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 23%|██▎ | 113/500 [00:38<01:04, 6.03it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "\u001b[32m2025-12-21 11:22:56.107\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n", "Example: {'pubid': 26686513, 'question': 'Cycloplegic autorefraction in young adults: is it mandatory?', 'context': {'contexts': ['The precise correction of refractive error is especially important in young adults. It is unclear whether cycloplegic refraction is necessary in this age group. The purpose of this study was to compare the non-cycloplegic and cycloplegic spherical equivalent (SE) refractive error measured in young adults.', 'This was a prospective study of 1400 eyes (n\\u2009=\\u2009700) of enlisted soldiers aged 18 to 21\\xa0years who were consecutively evaluated in an outpatient army ophthalmology clinic. One drop of cyclopentolate 1\\xa0% was installed twice 10\\xa0min apart, and cycloplegic refraction was performed in both eyes 40\\xa0min later using an auto-refractor. The difference between non-cycloplegic and cycloplegic refractive measurements was analyzed.', 'The mean difference in SE between non-cycloplegic and cycloplegic measurements was 0.68\\u2009±\\u20090.83\\xa0D (95\\xa0% CI, 0.64-0.72). Significantly greater differences were observed in hypermetropes than myopes (1.30\\u2009±\\u20090.90\\xa0D versus 0.46\\u2009±\\u20090.68\\xa0D, p\\u2009<\\u20090.001). Moderate hypermetropes (2 to 5\\xa0D) demonstrated significantly greater refractive error than mild (0.5 to 2\\xa0D) or severe (>5\\xa0D) hypermetropes (1.71\\u2009±\\u20091.18\\xa0D versus 1.19\\u2009±\\u20090.74\\xa0D and 1.16\\u2009±\\u20091.08\\xa0D respectively, p\\u2009<\\u20090.001).'], 'labels': ['PURPOSE', 'METHODS', 'RESULTS'], 'meshes': ['Adolescent', 'Cyclopentolate', 'Female', 'Humans', 'Hyperopia', 'Male', 'Military Personnel', 'Mydriatics', 'Myopia', 'Prospective Studies', 'Pupil', 'Refraction, Ocular', 'Retinoscopy', 'Young Adult'], 'reasoning_required_pred': ['m', 'a', 'y', 'b', 'e'], 'reasoning_free_pred': ['y', 'e', 's']}, 'long_answer': 'Young hypermetropic adults possessed +1 to +2\\xa0D of latent hypermetropia. In contrast, young myopic adults revealed pseudomyopia of -0.5\\xa0D. Cycloplegic refraction should be performed in young hypermetropic adults complaining of various signs of asthenopia.', 'final_decision': 'maybe'}\n", "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n", "\u001b[32m2025-12-21 11:22:56.160\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n", "Example: {'pubid': 21823940, 'question': 'Department of Transportation vs self-reported data on motor vehicle collisions and driving convictions for stroke survivors: do they agree?', 'context': {'contexts': [\"Research on stroke survivors' driving safety has typically used either self-reports or government records, but the extent to which the 2 may differ is not known. We compared government records and self-reports of motor vehicle collisions and driving convictions in a sample of stroke survivors.\", 'The 56 participants were originally recruited for a prospective study on driving and community re-integration post-stroke; the study population consisted of moderately impaired stroke survivors without severe communication disorders who had been referred for a driving assessment. The driving records of the 56 participants for the 5 years before study entry and the 1-year study period were acquired with written consent from the Ministry of Transportation of Ontario (MTO), Canada. Self-reports of collisions and convictions were acquired via a semistructured interview and then compared with the MTO records.', 'Forty-three participants completed the study. For 7 (13.5%) the MTO records did not match the self-reports regarding collision involvement, and for 9 (17.3%) the MTO records did not match self-reports regarding driving convictions. The kappa coefficient for the correlation between MTO records and self-reports was 0.52 for collisions and 0.47 for convictions (both in the moderate range of agreement). When both sources of data were consulted, up to 56 percent more accidents and up to 46 percent more convictions were identified in the study population in the 5 years before study entry compared to when either source was used alone.'], 'labels': ['OBJECTIVE', 'METHODS', 'RESULTS'], 'meshes': ['Accidents, Traffic', 'Aged', 'Automobile Driving', 'Female', 'Government Agencies', 'Humans', 'Male', 'Middle Aged', 'Ontario', 'Prospective Studies', 'Records as Topic', 'Reproducibility of Results', 'Safety', 'Self Report', 'Stroke', 'Survivors'], 'reasoning_required_pred': ['n', 'o'], 'reasoning_free_pred': ['n', 'o']}, 'long_answer': 'In our population of stroke survivors, self-reports of motor vehicle collisions and driving convictions differed from government records. In future studies, the use of both government and self-reported data would ensure a more accurate picture of driving safety post-stroke.', 'final_decision': 'no'}\n", "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 23%|██▎ | 116/500 [00:38<00:54, 7.08it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 23%|██▎ | 117/500 [00:39<01:20, 4.77it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "\u001b[32m2025-12-21 11:22:57.163\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n", "Example: {'pubid': 18322741, 'question': 'Does laparoscopic surgery decrease the risk of atrial fibrillation after foregut surgery?', 'context': {'contexts': ['Atrial fibrillation, which occurs in 12% of all major foregut surgeries, can prolong hospital stay and increase morbidity. Minimally invasive techniques in foregut surgery have been suggested to cause less tissue trauma. We examined the factors associated with new-onset atrial fibrillation after foregut surgery at our institution.', 'We retrospectively examined the records of 154 adult patients who underwent major foregut surgery which included esophagectomy, partial or total gastrectomy, redo Heller myotomy, redo or transthoracic fundoplications. Univariate and multivariate logistic regression analysis with standard modeling techniques were performed to determine risk factors for new-onset atrial fibrillation.', 'Of the 154 patients, 14 patients developed new-onset atrial fibrillation with a higher mean age of 67.1 years (+/-8.8 years) versus 56.4 years (+/-14.1 years) (p = 0.006). Laparoscopic (p = 0.004) and nonthoracic surgeries (p = 0.01) were associated with lower risk of atrial fibrillation. Patients with atrial fibrillation had received more fluid (6.5 +/- 2.8 liters versus 5.3 +/- 2.0 liters) and had longer operations (370 +/- 103 min versus 362 +/- 142 min), none of which were statistically significant. The average intensive care length of stay of patients was longer: 7.5 +/- 6.8 days versus 4.0 +/- 7.1 days (p = 0.004). Multivariate analysis revealed an association of atrial fibrillation with age (OR 1.08, 95% CI 1.02-1.14, p = 0.01), and laparoscopic surgery (OR 0.09, 95% CI 0.01-0.95, p = 0.04) after adjusting for surgery type.'], 'labels': ['BACKGROUND', 'METHODS', 'RESULTS'], 'meshes': ['Adult', 'Age Factors', 'Aged', 'Atrial Fibrillation', 'Cohort Studies', 'Digestive System Surgical Procedures', 'Female', 'Humans', 'Laparoscopy', 'Length of Stay', 'Male', 'Middle Aged', 'Retrospective Studies', 'Risk Factors'], 'reasoning_required_pred': ['y', 'e', 's'], 'reasoning_free_pred': ['y', 'e', 's']}, 'long_answer': 'Laparoscopic surgery is associated with lower risk of atrial fibrillation in foregut surgery. Development of atrial fibrillation is associated with increased length of intensive care stay. We recommend a prospective trial to confirm our findings.', 'final_decision': 'yes'}\n", "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 24%|██▍ | 120/500 [00:39<01:23, 4.54it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 24%|██▍ | 122/500 [00:39<01:00, 6.21it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 25%|██▌ | 126/500 [00:40<00:48, 7.67it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 26%|██▌ | 130/500 [00:40<00:39, 9.48it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 27%|██▋ | 134/500 [00:41<00:56, 6.52it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 27%|██▋ | 137/500 [00:42<01:10, 5.16it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 28%|██▊ | 139/500 [00:42<00:56, 6.39it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 29%|██▊ | 143/500 [00:43<00:44, 8.10it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 29%|██▉ | 145/500 [00:43<00:45, 7.80it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 30%|███ | 150/500 [00:43<00:33, 10.47it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 30%|███ | 152/500 [00:44<00:39, 8.77it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 31%|███ | 154/500 [00:44<00:35, 9.81it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 31%|███▏ | 157/500 [00:45<01:07, 5.07it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 32%|███▏ | 158/500 [00:45<01:08, 4.97it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 32%|███▏ | 159/500 [00:45<01:20, 4.24it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 32%|███▏ | 161/500 [00:46<01:19, 4.27it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 32%|███▏ | 162/500 [00:46<01:18, 4.30it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 33%|███▎ | 165/500 [00:47<01:15, 4.45it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 33%|███▎ | 166/500 [00:47<01:15, 4.42it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 34%|███▍ | 170/500 [00:47<00:47, 6.91it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 34%|███▍ | 171/500 [00:48<00:49, 6.66it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 35%|███▍ | 174/500 [00:48<00:48, 6.78it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 35%|███▌ | 176/500 [00:48<00:57, 5.66it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 35%|███▌ | 177/500 [00:49<01:22, 3.94it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 36%|███▌ | 178/500 [00:49<01:29, 3.61it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 36%|███▌ | 181/500 [00:49<00:53, 5.96it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 37%|███▋ | 184/500 [00:50<00:46, 6.84it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 37%|███▋ | 186/500 [00:50<00:53, 5.87it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 37%|███▋ | 187/500 [00:50<00:50, 6.24it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 38%|███▊ | 188/500 [00:51<01:14, 4.17it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 38%|███▊ | 191/500 [00:52<01:09, 4.44it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 39%|███▉ | 194/500 [01:00<07:17, 1.43s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 39%|███▉ | 196/500 [01:01<04:37, 1.09it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "\u001b[32m2025-12-21 11:23:18.933\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n", "Example: {'pubid': 26370095, 'question': 'Are financial incentives cost-effective to support smoking cessation during pregnancy?', 'context': {'contexts': ['To investigate the cost-effectiveness of up to £400 worth of financial incentives for smoking cessation in pregnancy as an adjunct to routine health care.', 'Cost-effectiveness analysis based on a Phase II randomized controlled trial (RCT) and a cost-utility analysis using a life-time Markov model.', 'The RCT was undertaken in Glasgow, Scotland. The economic analysis was undertaken from the UK National Health Service (NHS) perspective.', 'A total of 612 pregnant women randomized to receive usual cessation support plus or minus financial incentives of up to £400 vouchers (US $609), contingent upon smoking cessation.', 'Comparison of usual support and incentive interventions in terms of cotinine-validated quitters, quality-adjusted life years (QALYs) and direct costs to the NHS.', \"The incremental cost per quitter at 34-38 weeks pregnant was £1127 ($1716).This is similar to the standard look-up value derived from Stapleton&West's published ICER tables, £1390 per quitter, by looking up the Cessation in Pregnancy Incentives Trial (CIPT) incremental cost (£157) and incremental 6-month quit outcome (0.14). The life-time model resulted in an incremental cost of £17 [95% confidence interval (CI)\\u2009=\\u2009-£93, £107] and a gain of 0.04 QALYs (95% CI\\u2009=\\u2009-0.058, 0.145), giving an ICER of £482/QALY ($734/QALY). Probabilistic sensitivity analysis indicates uncertainty in these results, particularly regarding relapse after birth. The expected value of perfect information was £30 million (at a willingness to pay of £30\\u2009000/QALY), so given current uncertainty, additional research is potentially worthwhile.\"], 'labels': ['AIMS', 'DESIGN', 'SETTING', 'PARTICIPANTS', 'MEASUREMENTS', 'FINDINGS'], 'meshes': ['Cost-Benefit Analysis', 'Female', 'Health Promotion', 'Humans', 'Markov Chains', 'Motivation', 'Pregnancy', 'Pregnancy Complications', 'Prenatal Care', 'Quality-Adjusted Life Years', 'Scotland', 'Smoking', 'Smoking Cessation', 'Smoking Prevention'], 'reasoning_required_pred': ['m', 'a', 'y', 'b', 'e'], 'reasoning_free_pred': ['y', 'e', 's']}, 'long_answer': 'Financial incentives for smoking cessation in pregnancy are highly cost-effective, with an incremental cost per quality-adjusted life years of £482, which is well below recommended decision thresholds.', 'final_decision': 'yes'}\n", "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 40%|████ | 200/500 [01:01<02:18, 2.17it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 40%|████ | 202/500 [01:01<01:47, 2.77it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 41%|████ | 204/500 [01:01<01:29, 3.30it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 41%|████ | 206/500 [01:02<01:28, 3.32it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 42%|████▏ | 208/500 [01:02<01:24, 3.47it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 42%|████▏ | 211/500 [01:03<00:56, 5.11it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 43%|████▎ | 213/500 [01:03<00:41, 6.90it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 43%|████▎ | 215/500 [01:04<01:13, 3.90it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 43%|████▎ | 216/500 [01:04<01:18, 3.63it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 44%|████▍ | 221/500 [01:05<00:41, 6.68it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 45%|████▌ | 226/500 [01:05<00:34, 8.04it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 46%|████▌ | 228/500 [01:05<00:36, 7.55it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 46%|████▌ | 230/500 [01:06<00:33, 8.14it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 46%|████▋ | 232/500 [01:07<00:57, 4.64it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 47%|████▋ | 235/500 [01:07<00:52, 5.09it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 48%|████▊ | 238/500 [01:07<00:42, 6.21it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 49%|████▊ | 243/500 [01:08<00:29, 8.84it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 49%|████▉ | 245/500 [01:08<00:24, 10.45it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 49%|████▉ | 247/500 [01:09<00:49, 5.12it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 50%|████▉ | 249/500 [01:09<00:49, 5.11it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 50%|█████ | 250/500 [01:09<00:47, 5.23it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 51%|█████ | 253/500 [01:10<00:41, 5.98it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 51%|█████ | 255/500 [01:10<00:34, 7.08it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 52%|█████▏ | 259/500 [01:10<00:25, 9.43it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 52%|█████▏ | 261/500 [01:11<00:27, 8.59it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 53%|█████▎ | 263/500 [01:11<00:41, 5.67it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 53%|█████▎ | 265/500 [01:12<00:45, 5.16it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 54%|█████▍ | 270/500 [01:12<00:23, 9.71it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 54%|█████▍ | 272/500 [01:30<10:13, 2.69s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 55%|█████▍ | 273/500 [01:31<08:38, 2.28s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "\u001b[32m2025-12-21 11:23:49.170\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n", "Example: {'pubid': 22867778, 'question': \"Does responsibility affect the public's valuation of health care interventions?\", 'context': {'contexts': ['Health services often spend more on safety interventions than seems cost-effective. This study investigates whether the public value safety-related health care improvements more highly than the same improvements in contexts where the health care system is not responsible.', 'An online survey was conducted to elicit the relative importance placed on preventing harms caused by 1) health care (hospital-acquired infections, drug administration errors, injuries to health care staff), 2) individuals (personal lifestyle choices, sports-related injuries), and 3) nature (genetic disorders). Direct valuations were obtained from members of the public by using a person trade-off or \"matching\" method. Participants were asked to choose between two preventative interventions of equal cost and equal health benefit per person for the same number of people, but differing in causation. If participants indicated a preference, their strength of preference was measured by using person trade-off.', 'Responses were obtained from 1030 people, reflecting the sociodemographic mix of the UK population. Participants valued interventions preventing hospital-acquired infections (1.31) more highly than genetic disorders (1.0), although drug errors were valued similarly to genetic disorders (1.07), and interventions to prevent injury to health care staff were given less weight than genetic disorders (0.71). Less weight was also given to interventions related to lifestyle (0.65) and sports injuries (0.41).'], 'labels': ['OBJECTIVE', 'METHOD', 'RESULTS'], 'meshes': ['Adolescent', 'Adult', 'Athletic Injuries', 'Attitude to Health', 'Choice Behavior', 'Cost-Benefit Analysis', 'Cross Infection', 'Data Collection', 'Delivery of Health Care', 'Female', 'Genetic Diseases, Inborn', 'Harm Reduction', 'Humans', 'Internet', 'Life Style', 'Male', 'Medication Errors', 'Middle Aged', 'Occupational Injuries', 'Patient Preference', 'Patient Safety', 'Social Values', 'United Kingdom', 'Young Adult'], 'reasoning_required_pred': ['n', 'o'], 'reasoning_free_pred': ['m', 'a', 'y', 'b', 'e']}, 'long_answer': 'Our results suggest that people do not attach a simple fixed premium to \"safety-related\" interventions but that preferences depend more subtly on context. The use of the results of such public preference surveys to directly inform policy would therefore be premature.', 'final_decision': 'maybe'}\n", "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n", "\u001b[32m2025-12-21 11:23:49.816\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n", "Example: {'pubid': 11458136, 'question': 'Does managed care enable more low income persons to identify a usual source of care?', 'context': {'contexts': ['By requiring or encouraging enrollees to obtain a usual source of care, managed care programs hope to improve access to care without incurring higher costs.', '(1) To examine the effects of managed care on the likelihood of low-income persons having a usual source of care and a usual physician, and; (2) To examine the association between usual source of care and access.', 'Cross-sectional survey of households conducted during 1996 and 1997.', 'A nationally representative sample of 14,271 low-income persons.', 'Usual source of care, usual physician, managed care enrollment, managed care penetration.', 'High managed care penetration in the community is associated with a lower likelihood of having a usual source of care for uninsured persons (54.8% vs. 62.2% in low penetration areas) as well as a lower likelihood of having a usual physician (60% vs. 72.8%). Managed care has only marginal effects on the likelihood of having a usual source of care for privately insured and Medicaid beneficiaries. Having a usual physician substantially reduces unmet medical needs for the insured but less so for the uninsured.'], 'labels': ['BACKGROUND', 'OBJECTIVES', 'RESEARCH DESIGN', 'SUBJECTS', 'MEASURES', 'RESULTS'], 'meshes': ['Adult', 'Continuity of Patient Care', 'Cross-Sectional Studies', 'Female', 'Health Services Accessibility', 'Humans', 'Insurance Coverage', 'Male', 'Managed Care Programs', 'Medically Uninsured', 'Multivariate Analysis', 'Poverty', 'United States'], 'reasoning_required_pred': ['n', 'o'], 'reasoning_free_pred': ['m', 'a', 'y', 'b', 'e']}, 'long_answer': 'Having a usual physician can be an effective tool in improving access to care for low-income populations, although it is most effective when combined with insurance coverage. However, the effectiveness of managed care in linking more low-income persons to a medical home is uncertain, and may have unintended consequences for uninsured persons.', 'final_decision': 'maybe'}\n", "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 55%|█████▌ | 275/500 [01:31<06:19, 1.69s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m2025-12-21 11:23:49.995\u001b[0m | \u001b[31m\u001b[1mERROR \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 55%|█████▌ | 276/500 [01:32<05:14, 1.40s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "\u001b[32m2025-12-21 11:23:50.007\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n", "Example: {'pubid': 25819796, 'question': 'Literacy after cerebral hemispherectomy: Can the isolated right hemisphere read?', 'context': {'contexts': ['Cerebral hemispherectomy, a surgical procedure undergone to control intractable seizures, is becoming a standard procedure with more cases identified and treated early in life [33]. While the effect of the dominant hemisphere resection on spoken language has been extensively researched, little is known about reading abilities in individuals after left-sided resection. Left-lateralized phonological abilities are the key components of reading, i.e., grapheme-phoneme conversion skills [1]. These skills are critical for the acquisition of word-specific orthographic knowledge and have been shown to predict reading levels in average readers as well as in readers with mild cognitive disability [26]. Furthermore, impaired phonological processing has been implicated as the cognitive basis in struggling readers. Here, we explored the reading skills in participants who have undergone left cerebral hemispherectomy.', 'Seven individuals who have undergone left cerebral hemispherectomy to control intractable seizures associated with perinatal infarct have been recruited for this study. We examined if components of phonological processing that are shown to reliably separate average readers from struggling readers, i.e., phonological awareness, verbal memory, speed of retrieval, and size of vocabulary, show the same relationship to reading levels when they are mediated by the right hemisphere [2].', 'We found that about 60% of our group developed both word reading and paragraph reading in the average range. Phonological processing measured by both phonological awareness and nonword reading was unexpectedly spared in the majority of participants. Phonological awareness levels strongly correlated with word reading. Verbal memory, a component of phonological processing skills, together with receptive vocabulary size, positively correlated with reading levels similar to those reported in average readers. Receptive vocabulary, a bilateral function, was preserved to a certain degree similar to that of strongly left-lateralized phonological skills [3]. Later seizure onset was associated with better reading levels.'], 'labels': ['OBJECTIVES', 'METHODS', 'RESULTS'], 'meshes': ['Adolescent', 'Child', 'Cohort Studies', 'Drug Resistant Epilepsy', 'Female', 'Functional Laterality', 'Hemispherectomy', 'Humans', 'Literacy', 'Male', 'Postoperative Period', 'Reading', 'Young Adult'], 'reasoning_required_pred': ['y', 'e', 's'], 'reasoning_free_pred': ['y', 'e', 's']}, 'long_answer': 'When cerebral hemispherectomy is performed to control seizures associated with very early (in utero) insult, it has been found that the remaining right hemisphere is still able to support reading and phonological processing skills that are normally mediated by the left hemisphere. Our results also suggest the existence of variability in individuals after hemispherectomy, even within groups having the same etiology and similar timing of insult.', 'final_decision': 'yes'}\n", "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n", "\u001b[32m2025-12-21 11:23:50.083\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n", "Example: {'pubid': 21801416, 'question': 'The effect of an intracerebroventricular injection of metformin or AICAR on the plasma concentrations of melatonin in the ewe: potential involvement of AMPK?', 'context': {'contexts': ['It is now widely accepted that AMP-activated protein kinase (AMPK) is a critical regulator of energy homeostasis. Recently, it has been shown to regulate circadian clocks. In seasonal breeding species such as sheep, the circadian clock controls the secretion of an endogenous rhythm of melatonin and, as a consequence, is probably involved in the generation of seasonal rhythms of reproduction. Considering this, we identified the presence of the subunits of AMPK in different hypothalamic nuclei involved in the pre- and post-pineal pathways that control seasonality of reproduction in the ewe and we investigated if the intracerebroventricular (i.c.v.) injection of two activators of AMPK, metformin and AICAR, affected the circadian rhythm of melatonin in ewes that were housed in constant darkness. In parallel the secretion of insulin was monitored as a peripheral metabolic marker. We also investigated the effects of i.c.v. AICAR on the phosphorylation of AMPK and acetyl-CoA carboxylase (ACC), a downstream target of AMPK, in brain structures along the photoneuroendocrine pathway to the pineal gland.', 'All the subunits of AMPK that we studied were identified in all brain areas that were dissected but with some differences in their level of expression among structures. Metformin and AICAR both reduced (p<0.001 and p<0.01 respectively) the amplitude of the circadian rhythm of melatonin secretion independently of insulin secretion. The i.c.v. injection of AICAR only tended (p = 0.1) to increase the levels of phosphorylated AMPK in the paraventricular nucleus but significantly increased the levels of phosphorylated ACC in the paraventricular nucleus (p<0.001) and in the pineal gland (p<0.05).'], 'labels': ['BACKGROUND', 'RESULTS'], 'meshes': ['AMP-Activated Protein Kinases', 'Aminoimidazole Carboxamide', 'Animals', 'Brain', 'Circadian Rhythm', 'Female', 'Infusions, Intraventricular', 'Melatonin', 'Metformin', 'Ribonucleotides', 'Sheep'], 'reasoning_required_pred': ['y', 'e', 's'], 'reasoning_free_pred': ['y', 'e', 's']}, 'long_answer': 'Taken together, these results suggest a potential role for AMPK on the secretion of melatonin probably acting trough the paraventricular nucleus and/or directly in the pineal gland. We conclude that AMPK may act as a metabolic cue to modulate the rhythm of melatonin secretion.', 'final_decision': 'yes'}\n", "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n", "\u001b[32m2025-12-21 11:23:50.112\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n", "Example: {'pubid': 24019262, 'question': 'Does high blood pressure reduce the risk of chronic low back pain?', 'context': {'contexts': ['Epidemiological studies have suggested inverse relationships between blood pressure and prevalence of conditions such as migraine and headache. It is not yet clear whether similar relationships can be established for back pain in particular in prospective studies.', 'Associations between blood pressure and chronic low back pain were explored in the cross-sectional HUNT 2 survey of a Norwegian county in 1995-1997, including 39,872 individuals who never used antihypertensive medication. A prospective study, comprising 17,209 initially back pain-free individuals and 5740 individuals reporting low back pain, was established by re-examinations in the HUNT 3 survey in 2006-2008. Associations were assessed by logistic regression with respect to systolic, diastolic and pulse pressure, with adjustment for education, work status, physical activity, smoking, body mass and lipid levels.', 'In the cross-sectional study, all three blood pressure measures showed inverse relationships with prevalence of low back pain in both sexes. In the prospective study of disease-free women, baseline pulse pressure and systolic pressure were inversely associated with risk of low back pain [odds ratio (OR) 0.93 per 10\\u2009mm\\u2009Hg increase in pulse pressure, 95% confidence interval (CI) 0.89-0.98, p\\u2009=\\u20090.007; OR 0.95 per 10\\u2009mm Hg increase in systolic pressure, 95% CI 0.92-0.99, p\\u2009=\\u20090.005]. Results among men were equivocal. No associations were indicated with the occurrence of pain in individuals with low back pain at baseline.'], 'labels': ['BACKGROUND', 'METHODS', 'RESULTS'], 'meshes': ['Adult', 'Aged', 'Angiotensin Amide', 'Body Mass Index', 'Cross-Sectional Studies', 'Female', 'Genetic Testing', 'Humans', 'Logistic Models', 'Low Back Pain', 'Male', 'Middle Aged', 'Prevalence', 'Risk Factors'], 'reasoning_required_pred': ['m', 'a', 'y', 'b', 'e'], 'reasoning_free_pred': ['y', 'e', 's']}, 'long_answer': 'Results for low back pain are consistent with the theory of hypertension-associated hypalgesia, predicting diminished pain sensitivity with increasing blood pressure, possibly with modified reactions in people suffering from long-lasting pain.', 'final_decision': 'yes'}\n", "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 56%|█████▌ | 279/500 [01:32<02:54, 1.26it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m2025-12-21 11:23:50.117\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n", "Example: {'pubid': 18496363, 'question': 'Characterization of the gender dimorphism after injury and hemorrhagic shock: are hormonal differences responsible?', 'context': {'contexts': ['To characterize the gender dimorphism after injury with specific reference to the reproductive age of the women (young,<48 yrs of age, vs. old,>52 yrs of age) in a cohort of severely injured trauma patients for which significant variation in postinjury care is minimized.', 'Secondary data analysis of an ongoing prospective multicenter cohort study.', 'Academic, level I trauma and intensive care unit centers.', 'Blunt-injured adults with hemorrhagic shock.', 'None.', 'Separate Cox proportional hazard regression models were formulated based on all patients to evaluate the effects of gender on mortality, multiple organ failure, and nosocomial infection, after controlling for all important confounders. These models were then used to characterize the effect of gender in young and old age groups. Overall mortality, multiple organ failure, and nosocomial infection rates for the entire cohort (n = 1,036) were 20%, 40%, and 45%, respectively. Mean Injury Severity Score was 32 +/- 14 (mean +/- SD). Men (n = 680) and women (n = 356) were clinically similar except that men required higher crystalloid volumes, more often had a history of alcoholism and liver disease, and had greater ventilatory and intensive care unit requirements. Female gender was independently associated with a 43% and 23% lower risk of multiple organ failure and nosocomial infection, respectively. Gender remained an independent risk factor in young and old subgroup analysis, with the protection afforded by female gender remaining unchanged.'], 'labels': ['OBJECTIVE', 'DESIGN', 'SETTING', 'PATIENTS', 'INTERVENTIONS', 'MEASUREMENTS AND MAIN RESULTS'], 'meshes': ['Abbreviated Injury Scale', 'Adult', 'Age Factors', 'Cause of Death', 'Cohort Studies', 'Cross Infection', 'Estrogens', 'Female', 'Glasgow Coma Scale', 'Hospital Mortality', 'Humans', 'Injury Severity Score', 'Male', 'Menopause', 'Middle Aged', 'Multiple Organ Failure', 'Multiple Trauma', 'Proportional Hazards Models', 'Prospective Studies', 'Regression Analysis', 'Sex Factors', 'Shock, Hemorrhagic', 'Survival Analysis', 'Wounds, Nonpenetrating'], 'reasoning_required_pred': ['y', 'e', 's'], 'reasoning_free_pred': ['n', 'o']}, 'long_answer': 'The independent protective effect of female gender on multiple organ failure and nosocomial infection rates remains significant in both premenopausal and postmenopausal women when compared with similarly aged men. This is contrary to previous experimental studies and the known physiologic sex hormone changes that occur after menopause in women. These results suggest that factors other than sex hormones may be responsible for gender-based differences after injury.', 'final_decision': 'no'}\n", "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n", "\u001b[32m2025-12-21 11:23:50.152\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n", "Example: {'pubid': 22656647, 'question': 'Are acceptance rates of a national preventive home visit programme for older people socially imbalanced?', 'context': {'contexts': ['Preventive home visits are offered to community dwelling older people in Denmark aimed at maintaining their functional ability for as long as possible, but only two thirds of older people accept the offer from the municipalities. The purpose of this study is to investigate 1) whether socioeconomic status was associated with acceptance of preventive home visits among older people and 2) whether municipality invitational procedures for the preventive home visits modified the association.', 'The study population included 1,023 community dwelling 80-year-old individuals from the Danish intervention study on preventive home visits. Information on preventive home visit acceptance rates was obtained from questionnaires. Socioeconomic status was measured by financial assets obtained from national registry data, and invitational procedures were identified through the municipalities. Logistic regression analyses were used, adjusted by gender.', 'Older persons with high financial assets accepted preventive home visits more frequently than persons with low assets (adjusted OR = 1.5 (CI95%: 1.1-2.0)). However, the association was attenuated when adjusted by the invitational procedures. The odds ratio for accepting preventive home visits was larger among persons with low financial assets invited by a letter with a proposed date than among persons with high financial assets invited by other procedures, though these estimates had wide confidence intervals.'], 'labels': ['BACKGROUND', 'METHODS', 'RESULTS'], 'meshes': ['Aged, 80 and over', 'Cross-Sectional Studies', 'Denmark', 'Female', 'Financing, Personal', 'Geriatric Assessment', 'Health Services for the Aged', 'Healthcare Disparities', 'Home Care Services', 'House Calls', 'Humans', 'Logistic Models', 'Male', 'Patient Acceptance of Health Care', 'Physicians, Family', 'Prevalence', 'Preventive Health Services', 'Program Evaluation', 'Residence Characteristics', 'Sex Distribution', 'Social Class', 'Surveys and Questionnaires'], 'reasoning_required_pred': ['y', 'e', 's'], 'reasoning_free_pred': ['y', 'e', 's']}, 'long_answer': 'High socioeconomic status was associated with a higher acceptance rate of preventive home visits, but the association was attenuated by invitational procedures. The results indicate that the social inequality in acceptance of publicly offered preventive services might decrease if municipalities adopt more proactive invitational procedures.', 'final_decision': 'yes'}\n", "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 56%|█████▋ | 282/500 [01:32<01:54, 1.91it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 57%|█████▋ | 285/500 [01:33<01:33, 2.31it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "\u001b[32m2025-12-21 11:23:52.455\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n", "Example: {'pubid': 20363841, 'question': 'Do symptoms predict COPD in smokers?', 'context': {'contexts': ['The US Preventive Services Task Force recommends against spirometry in the absence of symptoms. However, as much as 50% of COPD cases in the United States remain undiagnosed.', 'Report of symptoms, smoking history, and spirometric data were collected from subjects screened for a work-related medical evaluation (N = 3,955). Prevalence of airflow obstruction and respiratory symptoms were assessed. Sensitivity, specificity, positive and negative predictive values, and relative risks of predicting symptoms and smoking history for COPD were calculated.', 'Forty-four percent of smokers in our sample had airways obstruction (AO). Of these, 36% reported a diagnosis of or treatment for COPD. Odds ratio (95% CI) for AO with smoking (>or = 20 pack-years) was 3.73 (3.12- 4.45), 1.98 (1.73-2.27) for cough, 1.79 (1.55-2.08) for dyspnea, 1.95 (1.70-2.34) for sputum, and 2.59 (2.26-2.97) for wheeze. Respiratory symptoms were reported by 92% of smokers with AO, 86% smokers with restriction, 76% smokers with normal spirometry, and 73% of nonsmokers. Sensitivity (92% vs 90%), specificity (19% vs 22%), positive (47% vs 40%) and negative (75% vs 80%) predictive values for the presence of one or more symptoms were similar between smokers and all subjects.'], 'labels': ['BACKGROUND', 'METHODS', 'RESULTS'], 'meshes': ['Aged', 'Female', 'Humans', 'Logistic Models', 'Male', 'Middle Aged', 'Predictive Value of Tests', 'Prevalence', 'Pulmonary Disease, Chronic Obstructive', 'Radiography, Thoracic', 'Respiratory Function Tests', 'Risk Factors', 'Sensitivity and Specificity', 'Smoking', 'Spirometry', 'Surveys and Questionnaires'], 'reasoning_required_pred': ['y', 'e', 's'], 'reasoning_free_pred': ['n', 'o']}, 'long_answer': 'COPD is underdiagnosed in the United States. Symptoms are frequent in subjects with AO and increase their risk for COPD, but add little beyond age and smoking history to the predictive value of spirometry. In view of the high prevalence of symptoms and their poor predictive value, a simpler and more effective approach would be to screen older smokers.', 'final_decision': 'no'}\n", "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 57%|█████▋ | 286/500 [01:34<01:59, 1.79it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m2025-12-21 11:23:52.484\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n", "Example: {'pubid': 21850494, 'question': 'Hepatorenal syndrome: are we missing some prognostic factors?', 'context': {'contexts': ['Hepatorenal syndrome (HRS) is the functional renal failure associated with advanced cirrhosis and has also been described in fulminant hepatic failure. Without liver transplantation its prognosis is dismal. Our study included patients with type 1 HRS associated with cirrhosis, who were not liver transplant candidates.AIM: To identify variables associated with improved survival.', 'Sixty-eight patients fulfilled the revised Ascites Club Criteria for type 1 HRS. None of them was suitable for liver transplantation. All the patients were treated with combinations of: albumin, midodrine and octreotide, pressors, and hemodialysis.', 'Median survival was 13 days for the whole group. Survival varied with the end-stage liver disease (ESLD) etiology: autoimmune, 49 days, cardiac cirrhosis, 22 days, idiopathic, 15.5 days, viral, 15 days, hepatitis C and alcohol, 14.5 days, alcohol 8 days, and neoplasia 4 days (p = 0.048). Survival of HRS associated with alcoholic liver disease versus other etiologies was not statistically significant (p = 0.1). Increased serum creatinine (p = 0.02) and urinary sodium 6-10 mEq/l (p = 0.027) at the initiation of therapy were prognostic factors for mortality. HRS treatment modalities (p = 0.73), use of dialysis (p = 0.56), dialysis modality (p = 0.35), use of vasopressors (p = 0.26), pre-existing renal disease (p = 0.49), gender (p = 0.90), and age (p = 0.57) were not associated with survival.'], 'labels': ['BACKGROUND', 'METHODS', 'RESULTS'], 'meshes': ['Albumins', 'Alcoholism', 'Autoimmune Diseases', 'Creatinine', 'End Stage Liver Disease', 'Female', 'Hepatitis C', 'Hepatorenal Syndrome', 'Humans', 'Liver Cirrhosis', 'Male', 'Middle Aged', 'Midodrine', 'Octreotide', 'Prognosis', 'Renal Dialysis', 'Retrospective Studies', 'Sodium', 'Survival Rate'], 'reasoning_required_pred': ['y', 'e', 's'], 'reasoning_free_pred': ['y', 'e', 's']}, 'long_answer': 'We report for the first time ESLD etiology as a prognostic factor for survival. The renal function (expressed as serum creatinine) and urinary Na (<5 mEq/l) at the time of diagnosis were found to be associated with survival, suggesting that early treatment might increase survival.', 'final_decision': 'yes'}\n", "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 58%|█████▊ | 289/500 [01:35<01:26, 2.44it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 59%|█████▊ | 293/500 [01:35<00:55, 3.73it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 59%|█████▉ | 297/500 [01:36<00:32, 6.18it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 60%|█████▉ | 299/500 [01:36<00:27, 7.22it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "\u001b[32m2025-12-21 11:23:54.686\u001b[0m | \u001b[31m\u001b[1mERROR \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 60%|██████ | 301/500 [01:37<00:41, 4.76it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 61%|██████ | 303/500 [01:37<00:33, 5.95it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 61%|██████ | 305/500 [01:37<00:40, 4.84it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 61%|██████ | 306/500 [01:37<00:41, 4.67it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 61%|██████▏ | 307/500 [01:38<00:44, 4.37it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 62%|██████▏ | 309/500 [01:38<00:36, 5.19it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 63%|██████▎ | 313/500 [01:39<00:38, 4.87it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 63%|██████▎ | 315/500 [01:39<00:31, 5.80it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 64%|██████▍ | 320/500 [01:40<00:20, 8.69it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 64%|██████▍ | 322/500 [01:40<00:19, 8.97it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 65%|██████▍ | 324/500 [01:40<00:25, 6.84it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 65%|██████▌ | 326/500 [01:41<00:35, 4.96it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 65%|██████▌ | 327/500 [01:41<00:36, 4.68it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 66%|██████▌ | 329/500 [01:42<00:36, 4.73it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 66%|██████▌ | 331/500 [01:42<00:40, 4.14it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 67%|██████▋ | 333/500 [01:43<00:29, 5.62it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 67%|██████▋ | 335/500 [01:43<00:23, 7.17it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 68%|██████▊ | 338/500 [01:43<00:23, 6.96it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 68%|██████▊ | 340/500 [01:44<00:26, 6.13it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 68%|██████▊ | 342/500 [01:44<00:31, 5.00it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 69%|██████▉ | 344/500 [01:44<00:28, 5.48it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 69%|██████▉ | 345/500 [01:45<00:30, 5.03it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 69%|██████▉ | 346/500 [01:45<00:31, 4.83it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 70%|███████ | 351/500 [01:45<00:17, 8.36it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 70%|███████ | 352/500 [01:46<00:24, 6.12it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 71%|███████ | 354/500 [01:46<00:22, 6.58it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 71%|███████▏ | 357/500 [01:47<00:24, 5.84it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 72%|███████▏ | 358/500 [01:47<00:24, 5.84it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 72%|███████▏ | 360/500 [01:47<00:22, 6.31it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 72%|███████▏ | 362/500 [01:47<00:23, 5.79it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 73%|███████▎ | 365/500 [01:48<00:22, 5.98it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 73%|███████▎ | 367/500 [01:48<00:17, 7.44it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 74%|███████▎ | 368/500 [01:48<00:20, 6.43it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 74%|███████▍ | 370/500 [01:49<00:21, 6.19it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 75%|███████▍ | 373/500 [01:49<00:16, 7.74it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 75%|███████▌ | 375/500 [01:49<00:13, 9.48it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 76%|███████▌ | 379/500 [01:50<00:22, 5.34it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 76%|███████▌ | 381/500 [01:50<00:18, 6.28it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 77%|███████▋ | 383/500 [01:51<00:16, 6.91it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 77%|███████▋ | 385/500 [01:51<00:26, 4.30it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 78%|███████▊ | 389/500 [01:52<00:18, 5.93it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 78%|███████▊ | 392/500 [02:00<02:02, 1.14s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 79%|███████▉ | 394/500 [02:01<01:23, 1.26it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "\u001b[32m2025-12-21 11:24:19.414\u001b[0m | \u001b[31m\u001b[1mERROR \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 79%|███████▉ | 397/500 [02:01<00:43, 2.36it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "\u001b[32m2025-12-21 11:24:19.466\u001b[0m | \u001b[31m\u001b[1mERROR \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "\u001b[32m2025-12-21 11:24:19.605\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n", "Example: {'pubid': 11759976, 'question': 'Advanced epithelial ovarian carcinoma in Thai women: should we continue to offer second-look laparotomy?', 'context': {'contexts': ['To determine survival among patients with epithelial ovarian carcinoma (EOC) who underwent a second-look laparotomy (SLL) and those refusing the procedure. Also to analyze factor(s) influencing the survival of the patients.', 'Medical records were reviewed of patients with advanced EOC who were clinically free of disease after primary surgery and platinum-based chemotherapy between January 1, 1992, and December 31, 1998. All of them were offered SLL. Measurement outcomes include patient survival and disease-free survival.', \"There were 50 patients with clinically complete remission after chemotherapy. Sixteen patients underwent SLL, and thirty-four patients refused the procedure (NSLL). Seven patients (43.8%) were reported to have positive SLL. After the median follow-up time of 35 months, 12 patients had died, and 5 patients were lost to follow-up. The median survival time for patients with SLL was about 60 months. Five-year survival rates of patients in the SLL, and NSLL groups were 37 per cent (95%CI = 7%-69%), and 88 per cent (95%CI = 65%-96%) respectively (P<0.001). The median time to relapse was about 25 months for patients with negative SLL. Five-year disease-free survival rates of patients in the negative SLL, and NSLL groups were 28 per cent (95%CI = 4%-59%), and 54 per cent (95%CI = 34%-70%) respectively (P=0.251). By Cox regression analysis, tumor grade was the only significant prognostic factor influencing patients' survival (HR = 6, 95%CI of HR = 1.2-34.2).\"], 'labels': ['OBJECTIVE', 'METHOD AND MATERIAL', 'RESULTS'], 'meshes': ['Chemotherapy, Adjuvant', 'Disease-Free Survival', 'Female', 'Humans', 'Incidence', 'Neoplasm Recurrence, Local', 'Ovarian Neoplasms', 'Proportional Hazards Models', 'Second-Look Surgery', 'Survival Rate', 'Thailand'], 'reasoning_required_pred': ['m', 'a', 'y', 'b', 'e'], 'reasoning_free_pred': ['n', 'o']}, 'long_answer': \"The second-look laparotomy doesn't have a favorable impact on overall and disease-free survival. Tumor grade is the only independent prognostic variable for survival of the patients.\", 'final_decision': 'no'}\n", "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 80%|███████▉ | 399/500 [02:01<00:31, 3.16it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 80%|████████ | 401/500 [02:01<00:24, 4.04it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 81%|████████ | 404/500 [02:03<00:31, 3.07it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 81%|████████ | 405/500 [02:03<00:28, 3.34it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 81%|████████ | 406/500 [02:03<00:28, 3.30it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 82%|████████▏ | 408/500 [02:04<00:22, 4.05it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 82%|████████▏ | 409/500 [02:04<00:25, 3.63it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 82%|████████▏ | 410/500 [02:04<00:23, 3.76it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 82%|████████▏ | 411/500 [02:05<00:27, 3.21it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 83%|████████▎ | 415/500 [02:05<00:16, 5.29it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 83%|████████▎ | 417/500 [02:05<00:11, 7.21it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 84%|████████▍ | 420/500 [02:05<00:07, 10.00it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 84%|████████▍ | 422/500 [02:06<00:09, 8.30it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 85%|████████▍ | 424/500 [02:07<00:14, 5.22it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 85%|████████▌ | 427/500 [02:07<00:11, 6.15it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 86%|████████▌ | 429/500 [02:07<00:09, 7.44it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 86%|████████▌ | 431/500 [02:07<00:09, 7.54it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 87%|████████▋ | 433/500 [02:08<00:10, 6.64it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 87%|████████▋ | 434/500 [02:08<00:10, 6.59it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 88%|████████▊ | 438/500 [02:08<00:07, 7.93it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 88%|████████▊ | 441/500 [02:09<00:06, 8.88it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 89%|████████▉ | 444/500 [02:09<00:06, 9.14it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 89%|████████▉ | 445/500 [02:09<00:08, 6.40it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 89%|████████▉ | 446/500 [02:10<00:09, 5.52it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 90%|████████▉ | 449/500 [02:10<00:09, 5.42it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 90%|█████████ | 451/500 [02:10<00:08, 5.97it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 91%|█████████ | 454/500 [02:11<00:06, 7.23it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 92%|█████████▏| 459/500 [02:11<00:03, 11.98it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 92%|█████████▏| 461/500 [02:12<00:04, 7.89it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 93%|█████████▎| 463/500 [02:12<00:05, 6.70it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 93%|█████████▎| 465/500 [02:31<01:36, 2.76s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 93%|█████████▎| 466/500 [02:31<01:14, 2.20s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "\u001b[32m2025-12-21 11:24:49.268\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n", "Example: {'pubid': 12684740, 'question': 'Cue-induced behavioural activation: a novel model of alcohol craving?', 'context': {'contexts': ['Alcohol-associated cues elicit craving in human addicts but little is known about craving mechanisms. Current animal models focus on relapse and this may confound the effect of environmental cues. OBJECTIVES. To develop a model to study the effects of environmental cues on alcohol consumption in animals not experiencing withdrawal or relapse.', 'Rats were trained to orally self-administer an alcohol (5% w/v)/saccharin (0.2%) solution 30 min a day for 20 days. After stable responding on a free choice between alcohol/saccharin and water, rats were exposed to 5, 10 or 15 min of alcohol-associated cues or 5 min of non-alcohol associated cues. The effect of a 5-min cue was measured after a 10-day break from training or pre-treatment with 0.03, 0.1 or 1 mg/kg naltrexone.', 'Rats given 5 min of alcohol-associated cues responded significantly more on the active lever (26% increase) and consumed more alcohol as verified by increased blood alcohol levels (8.9 mM versus control 7.5 mM). Ten or 15 min of cues did not change alcohol consumption and 5 min in a novel environment decreased response by 66%. After a 10-day break in training, 5 min of alcohol-associated cues still increased alcohol consumption (29% increase) and the cue effect could be dose-dependently blocked by naltrexone (143% decrease at 0.03 mg/kg).'], 'labels': ['RATIONALE', 'METHODS', 'RESULTS'], 'meshes': ['Alcohol Drinking', 'Animals', 'Behavior, Addictive', 'Cues', 'Discrimination Learning', 'Environment', 'Ethanol', 'Male', 'Models, Psychological', 'Naltrexone', 'Narcotic Antagonists', 'Rats', 'Rats, Wistar', 'Reinforcement (Psychology)', 'Self Administration', 'Time Factors'], 'reasoning_required_pred': ['y', 'e', 's'], 'reasoning_free_pred': ['y', 'e', 's']}, 'long_answer': 'Cue-induced behavioural activation was specific to alcohol cues, reproducible, persistent and could be blocked by naltrexone, and its correlation with human self-report of craving makes it a potentially useful model for studying alcohol craving.', 'final_decision': 'yes'}\n", "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 94%|█████████▍| 469/500 [02:31<00:35, 1.14s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "\u001b[32m2025-12-21 11:24:50.076\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n", "Example: {'pubid': 23337545, 'question': 'Is acute fibrinous and organizing pneumonia the expression of immune dysregulation?', 'context': {'contexts': ['Acute fibrinous and organizing pneumonia (AFOP) is a recently described histologic pattern of diffuse pulmonary disease. In children, all cases reported to date have been fatal. In this study, we describe the first nonfatal AFOP in a child and review the literature.', 'A 10-year-old boy developed very severe aplastic anemia (VSAA) after being admitted to our hospital with a fulminant hepatic failure of unknown origin. A chest computed tomography scan revealed multiple lung nodules and a biopsy of a pulmonary lesion showed all the signs of AFOP. Infectious workup remained negative. We started immunosuppressive therapy with antithymocyte globulin and cyclosporine to treat VSAA. Subsequent chest computed tomography scans showed a considerable diminution of the lung lesions but the VSAA did not improve until we performed hematopoietic stem cell transplantation 5 months later.'], 'labels': ['INTRODUCTION', 'DESCRIPTION'], 'meshes': ['Acute Disease', 'Child', 'Cryptogenic Organizing Pneumonia', 'Hematopoietic Stem Cell Transplantation', 'Humans', 'Immune System Diseases', 'Immunosuppressive Agents', 'Male'], 'reasoning_required_pred': ['y', 'e', 's'], 'reasoning_free_pred': ['y', 'e', 's']}, 'long_answer': 'Aplastic anemia is associated with a variety of autoimmune syndromes. The sequence of events in our patient suggests that the hepatic failure, AFOP, and the VSAA may all have been part of an autoimmune syndrome. AFOP could be the result of immune dysregulation in this pediatric case with favorable outcome after immunosuppressive therapy and hematopoietic stem cell transplantation.', 'final_decision': 'yes'}\n", "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 94%|█████████▍| 471/500 [02:32<00:26, 1.11it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m2025-12-21 11:24:50.114\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n", "Example: {'pubid': 27050505, 'question': 'Does angiotensin-converting enzyme-1 (ACE-1) gene polymorphism lead to chronic kidney disease among hypertensive patients?', 'context': {'contexts': ['Hypertension is one of the important contributing factors linked with both causation and development of kidney disease. It is a multifactorial, polygenic, and complex disorder due to interaction of several risk genes with environmental factors. The present study was aimed to explore genetic polymorphism in ACE-1 gene as a risk factor for CKD among hypertensive patients.', 'Three hundred patients were enrolled in the study. Ninety were hypertensive patients with CKD taken as cases, whereas 210 hypertensive patients without CKD were taken as controls. Demographic data including age, sex, Body mass index (BMI), and other risk factors were also recorded. DNA was extracted from blood by salting out method. Genotyping of ACE gene was done by PCR technique. All the statistical analysis was done by using Epi Info and SPSS version 16 software (SPSS Inc., Chicago, IL).', 'Mean age was higher in the control group (p\\u2009<\\u20090.05). Variables among two groups were compared out of which age, BMI, hemoglobin (Hb) was found to be statistically significant whereas other variables like systolic blood pressure, triglyceride and low-density lipoprotein were not. Blood urea and serum creatinine levels were statistically significant in the two genotypes (p\\u2009<\\u20090.05). Total and HDL cholesterol were statistically significant for DD genotype of ACE gene (OR\\u2009=\\u20091.42, 95% CI\\u2009=\\u20090.72-2.81). Similarly, the risk for CKD among hypertensive patients was also associated with D allele of ACE gene (OR\\u2009=\\u20091.25, 95% CI\\u2009=\\u20090.86-1.79).'], 'labels': ['BACKGROUND', 'METHODS', 'RESULTS'], 'meshes': ['Adult', 'Female', 'Gene-Environment Interaction', 'Genetic Predisposition to Disease', 'Humans', 'Hypertension', 'Kidney Function Tests', 'Male', 'Middle Aged', 'Peptidyl-Dipeptidase A', 'Polymorphism, Genetic', 'Renal Insufficiency, Chronic'], 'reasoning_required_pred': ['y', 'e', 's'], 'reasoning_free_pred': ['y', 'e', 's']}, 'long_answer': 'It is concluded that ACE-DD genotype may be a risk factor for the causation and development of chronic kidney failure among hypertensive patients.', 'final_decision': 'yes'}\n", "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 95%|█████████▌| 475/500 [02:32<00:12, 2.05it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 95%|█████████▌| 477/500 [02:32<00:08, 2.82it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "\u001b[32m2025-12-21 11:24:50.768\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n", "Example: {'pubid': 23375036, 'question': 'An HIV1/2 point of care test on sputum for screening TB/HIV co-infection in Central India - Will it work?', 'context': {'contexts': ['To determine whether the OraQuick® HIV-1/2 Assay (OraSure Technologies, Inc., Bethlehem, PA, USA) in sputum is a valid tool for HIV surveillance among TB patients.', 'A cross sectional study was carried out on sputa of patients diagnosed with tuberculosis. Sputa were tested for antibodies to HIV using OraQuick® HIV-1/2 Assay (OraSure Technologies, Inc., Bethlehem, PA, USA). The results were compared with results of serum ELISA.', 'Compared to serum ELISA, the OraQuick® HIV-1/2 Assay in sputum specimens reported 90% sensitivity (9/10) and 100% specificity (307/307), with a positive predictive value of 100% (95%CI: 66.37%-100.00%) and a negative predictive value of 99.68% (95%CI: 98.20%-99.99%).'], 'labels': ['OBJECTIVE', 'METHODS', 'RESULTS'], 'meshes': ['Adolescent', 'Adult', 'Aged', 'Cross-Sectional Studies', 'Enzyme-Linked Immunosorbent Assay', 'Female', 'HIV Infections', 'HIV-1', 'HIV-2', 'Humans', 'India', 'Male', 'Mass Screening', 'Middle Aged', 'Point-of-Care Systems', 'Sensitivity and Specificity', 'Sputum', 'Tuberculosis, Pulmonary', 'Young Adult'], 'reasoning_required_pred': ['y', 'e', 's'], 'reasoning_free_pred': ['y', 'e', 's']}, 'long_answer': 'This testing method may provide a useful strategy for conducting HIV surveillance in possible co-infected TB patients at peripheral centres. Since there is no investment on infrastructure, it may be possible for paramedical health professionals to carry out the test, particularly in areas with low HIV endemicity.', 'final_decision': 'yes'}\n", "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n", "\u001b[32m2025-12-21 11:24:51.102\u001b[0m | \u001b[31m\u001b[1mERROR \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 96%|█████████▌| 479/500 [02:33<00:06, 3.24it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "\u001b[32m2025-12-21 11:24:51.304\u001b[0m | \u001b[31m\u001b[1mERROR \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 96%|█████████▌| 481/500 [02:33<00:04, 3.81it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 97%|█████████▋| 483/500 [02:33<00:03, 4.56it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "\u001b[32m2025-12-21 11:24:51.907\u001b[0m | \u001b[31m\u001b[1mERROR \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 97%|█████████▋| 485/500 [02:34<00:02, 5.54it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 97%|█████████▋| 486/500 [02:34<00:02, 4.93it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "\u001b[32m2025-12-21 11:24:52.245\u001b[0m | \u001b[31m\u001b[1mERROR \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 98%|█████████▊| 488/500 [02:35<00:03, 3.24it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 98%|█████████▊| 489/500 [02:35<00:03, 3.22it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 98%|█████████▊| 490/500 [02:35<00:03, 3.24it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 98%|█████████▊| 492/500 [02:36<00:02, 3.51it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 99%|█████████▊| 493/500 [02:36<00:01, 4.08it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 99%|█████████▉| 497/500 [02:36<00:00, 7.50it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 100%|█████████▉| 499/500 [02:37<00:00, 4.29it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 100%|██████████| 500/500 [02:54<00:00, 2.87it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 1%| | 3/500 [00:02<04:44, 1.75it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 2%|▏ | 9/500 [00:02<01:19, 6.15it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 3%|▎ | 14/500 [00:02<00:46, 10.55it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 3%|▎ | 17/500 [00:02<00:42, 11.25it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 4%|▍ | 19/500 [00:04<01:33, 5.15it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 4%|▍ | 21/500 [00:04<01:37, 4.93it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 5%|▌ | 25/500 [00:05<01:17, 6.13it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 6%|▌ | 28/500 [00:05<01:21, 5.77it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 7%|▋ | 33/500 [00:05<00:52, 8.90it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 7%|▋ | 35/500 [00:06<01:05, 7.11it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 8%|▊ | 39/500 [00:07<01:09, 6.65it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 8%|▊ | 41/500 [00:07<01:30, 5.08it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 9%|▉ | 45/500 [00:08<01:06, 6.79it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 9%|▉ | 46/500 [00:08<01:06, 6.83it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 10%|█ | 50/500 [00:08<00:54, 8.25it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 10%|█ | 51/500 [00:08<00:55, 8.16it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 11%|█ | 54/500 [00:09<00:58, 7.60it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 11%|█ | 55/500 [00:09<01:10, 6.32it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 12%|█▏ | 58/500 [00:09<01:14, 5.92it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 12%|█▏ | 59/500 [00:10<01:33, 4.69it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 12%|█▏ | 62/500 [00:10<01:06, 6.62it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 13%|█▎ | 64/500 [00:10<01:07, 6.44it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 13%|█▎ | 67/500 [00:11<00:47, 9.08it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 14%|█▍ | 69/500 [00:11<01:00, 7.16it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 14%|█▍ | 71/500 [00:11<00:59, 7.25it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 15%|█▍ | 74/500 [00:12<00:57, 7.46it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 15%|█▌ | 75/500 [00:12<01:27, 4.85it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 16%|█▌ | 78/500 [00:12<01:01, 6.89it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 16%|█▌ | 79/500 [00:13<01:11, 5.86it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 16%|█▋ | 82/500 [00:13<01:01, 6.84it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 17%|█▋ | 84/500 [00:13<01:05, 6.33it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 17%|█▋ | 86/500 [00:14<01:10, 5.88it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 18%|█▊ | 89/500 [00:14<00:52, 7.76it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 18%|█▊ | 91/500 [00:14<00:40, 10.00it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 19%|█▊ | 93/500 [00:15<01:01, 6.64it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 19%|█▉ | 96/500 [00:15<00:57, 7.08it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 20%|██ | 101/500 [00:16<00:46, 8.56it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 21%|██ | 103/500 [00:16<01:05, 6.04it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 21%|██ | 106/500 [00:17<01:20, 4.89it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 22%|██▏ | 110/500 [00:17<00:52, 7.48it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 22%|██▏ | 112/500 [00:18<00:43, 8.97it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 23%|██▎ | 114/500 [00:18<00:37, 10.34it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 23%|██▎ | 116/500 [00:18<00:51, 7.52it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 24%|██▍ | 119/500 [00:19<01:01, 6.19it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 24%|██▍ | 120/500 [00:19<01:03, 5.99it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 24%|██▍ | 122/500 [00:19<00:56, 6.74it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 25%|██▌ | 125/500 [00:20<01:03, 5.87it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 26%|██▌ | 128/500 [00:20<00:53, 6.89it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 26%|██▌ | 131/500 [00:20<00:35, 10.50it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 27%|██▋ | 133/500 [00:21<00:56, 6.50it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 27%|██▋ | 135/500 [00:21<00:51, 7.02it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 27%|██▋ | 137/500 [00:21<00:53, 6.80it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 28%|██▊ | 139/500 [00:22<01:01, 5.88it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 28%|██▊ | 141/500 [00:22<00:46, 7.72it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 29%|██▊ | 143/500 [00:22<00:50, 7.07it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 29%|██▉ | 146/500 [00:22<00:39, 8.89it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 30%|██▉ | 148/500 [00:23<00:37, 9.30it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 30%|███ | 152/500 [00:23<00:47, 7.30it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 31%|███ | 155/500 [00:24<00:45, 7.53it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 31%|███ | 156/500 [00:24<00:49, 6.92it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 32%|███▏ | 159/500 [00:25<00:59, 5.70it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 32%|███▏ | 160/500 [00:36<14:43, 2.60s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 33%|███▎ | 163/500 [00:37<07:19, 1.31s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "\u001b[32m2025-12-21 11:25:49.625\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n", "Example: {'pubid': 11970923, 'question': 'Convulsions and retinal haemorrhage: should we look further?', 'context': {'contexts': ['The prevalence of retinal haemorrhages after convulsions is not well established. As these haemorrhages are considered characteristic of child abuse, we investigated their occurrence after convulsive episodes to see whether the finding of haemorrhage should prompt further investigation.', 'Prospective study of 153 children (aged 2 months to 2 years), seen in the emergency department after a convulsive episode. After a thorough history and physical examination, a retinal examination was performed by an ophthalmologist. If findings were positive, further investigation was undertaken to rule out systemic disorder or child abuse.', 'One child was found with unilateral retinal haemorrhages following an episode of a simple febrile convulsion. A thorough investigation uncovered no other reason for this finding.'], 'labels': ['BACKGROUND AND AIMS', 'METHODS', 'RESULTS'], 'meshes': ['Child Abuse', 'Child, Preschool', 'Diagnosis, Differential', 'Female', 'Humans', 'Infant', 'Male', 'Ophthalmoscopy', 'Physical Examination', 'Prospective Studies', 'Referral and Consultation', 'Retinal Hemorrhage', 'Seizures'], 'reasoning_required_pred': ['m', 'a', 'y', 'b', 'e'], 'reasoning_free_pred': ['y', 'e', 's']}, 'long_answer': 'Retinal haemorrhages following a convulsive episode are rare. Such a finding should trigger an extensive search for other reasons, including child abuse.', 'final_decision': 'yes'}\n", "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 33%|███▎ | 167/500 [00:37<03:28, 1.60it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m2025-12-21 11:25:50.121\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n", "Example: {'pubid': 16428354, 'question': 'Does rural or urban residence make a difference to neonatal outcome in premature birth?', 'context': {'contexts': ['Patients living in rural areas may be at a disadvantage in accessing tertiary health care.AIM: To test the hypothesis that very premature infants born to mothers residing in rural areas have poorer outcomes than those residing in urban areas in the state of New South Wales (NSW) and the Australian Capital Territory (ACT) despite a coordinated referral and transport system.', '\"Rural\" or \"urban\" status was based on the location of maternal residence. Perinatal characteristics, major morbidity and case mix adjusted mortality were compared between 1879 rural and 6775 urban infants<32 weeks gestational age, born in 1992-2002 and admitted to all 10 neonatal intensive care units in NSW and ACT.', 'Rural mothers were more likely to be teenaged, indigenous, and to have had a previous premature birth, prolonged ruptured membrane, and antenatal corticosteroid. Urban mothers were more likely to have had assisted conception and a caesarean section. More urban (93% v 83%) infants were born in a tertiary obstetric hospital. Infants of rural residence had a higher mortality (adjusted odds ratio (OR) 1.26, 95% confidence interval (CI) 1.07 to 1.48, p = 0.005). This trend was consistently seen in all subgroups and significantly for the tertiary hospital born population and the 30-31 weeks gestation subgroup. Regional birth data in this gestational age range also showed a higher stillbirth rate among rural infants (OR 1.20, 95% CI 1.09 to 1.32, p<0.001).'], 'labels': ['BACKGROUND', 'METHODS', 'RESULTS'], 'meshes': ['Australian Capital Territory', 'Epidemiologic Methods', 'Female', 'Gestational Age', 'Humans', 'Infant, Newborn', 'Infant, Premature', 'New South Wales', 'Pregnancy', 'Pregnancy Outcome', 'Premature Birth', 'Residence Characteristics', 'Rural Health', 'Stillbirth', 'Urban Health'], 'reasoning_required_pred': ['y', 'e', 's'], 'reasoning_free_pred': ['y', 'e', 's']}, 'long_answer': 'Premature births from rural mothers have a higher risk of stillbirth and mortality in neonatal intensive care than urban infants.', 'final_decision': 'yes'}\n", "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 34%|███▍ | 169/500 [00:38<02:54, 1.90it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "\u001b[32m2025-12-21 11:25:50.286\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n", "Example: {'pubid': 12846929, 'question': 'Quality of life in lung cancer patients: does socioeconomic status matter?', 'context': {'contexts': [\"As part of a prospective study on quality of life in newly diagnosed lung cancer patients an investigation was carried out to examine whether there were differences among patients' quality of life scores and their socioeconomic status.\", 'Quality of life was measured at two points in time (baseline and three months after initial treatment) using three standard instruments; the Nottingham Health Profile (NHP), the European Organization for Research and Cancer Treatment Quality of Life Questionnaire (EORTC QLQ-C30) and its lung cancer supplement (QLQ-LC13). Socioeconomic status for each individual patient was derived using Carstairs and Morris Deprivation Category ranging from 1 (least deprived) to 7 (most deprived) on the basis of the postcode sector of their address.', 'In all, 129 lung cancer patients entered into the study. Of these data for 82 patients were complete (at baseline and follow-up). 57% of patients were of lower socioeconomic status and they had more health problems, less functioning, and more symptoms as compared to affluent patients. Of these, physical mobility (P = 0.05), energy (P = 0.01), role functioning (P = 0.04), physical functioning (P = 0.03), and breathlessness (P = 0.02) were significant at baseline. However, at follow-up assessment there was no significant difference between patient groups nor did any consistent pattern emerge.'], 'labels': ['BACKGROUND', 'METHODS', 'RESULTS'], 'meshes': ['Aged', 'Female', 'Humans', 'Lung Neoplasms', 'Male', 'Middle Aged', 'Prospective Studies', 'Quality of Life', 'Sickness Impact Profile', 'Social Class', 'Statistics, Nonparametric', 'Surveys and Questionnaires'], 'reasoning_required_pred': ['y', 'e', 's'], 'reasoning_free_pred': ['y', 'e', 's']}, 'long_answer': \"At baseline assessment patients of lower socioeconomic status showed lower health related quality of life. Since there was no clear trend at follow-up assessment this suggests that patients from different socioeconomic status responded to treatment similarly. In general, the findings suggest that quality of life is not only the outcome of the disease and its treatment, but is also highly dependent on each patients' socioeconomic characteristics.\", 'final_decision': 'yes'}\n", "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n", "\u001b[32m2025-12-21 11:25:50.310\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n", "Example: {'pubid': 1571683, 'question': 'Storage of vaccines in the community: weak link in the cold chain?', 'context': {'contexts': ['To assess quality of storage of vaccines in the community.', 'Questionnaire survey of general practices and child health clinics, and monitoring of storage temperatures of selected refrigerators.', 'Central Manchester and Bradford health districts.', '45 general practices and five child health clinics, of which 40 (80%) responded. Eight practices were selected for refrigeration monitoring.', 'Adherence to Department of Health guidelines for vaccine storage, temperature range to which vaccines were exposed over two weeks.', 'Of the 40 respondents, only 16 were aware of the appropriate storage conditions for the vaccines; eight had minimum and maximum thermometers but only one of these was monitored daily. In six of the eight practices selected for monitoring of refrigeration temperatures the vaccines were exposed to either subzero temperatures (three fridges) or temperatures up to 16 degrees C (three). Two of these were specialised drug storage refrigerators with an incorporated thermostat and external temperature gauges.'], 'labels': ['OBJECTIVE', 'DESIGN', 'SETTING', 'SUBJECTS', 'MAIN OUTCOME MEASURES', 'RESULTS'], 'meshes': ['Child', 'Child Health Services', 'Drug Storage', 'Family Practice', 'Humans', 'Refrigeration', 'Time Factors', 'Vaccines'], 'reasoning_required_pred': ['n', 'o'], 'reasoning_free_pred': ['m', 'a', 'y', 'b', 'e']}, 'long_answer': 'Vaccines were exposed to temperatures that may reduce their potency. Safe storage of vaccines in the clinics cannot be ensured without adhering to the recommended guidelines. Provision of adequate equipment and training for staff in maintaining the \"cold chain\" and the use and care of equipment are important components of a successful immunisation programme.', 'final_decision': 'maybe'}\n", "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 34%|███▍ | 171/500 [00:38<02:11, 2.50it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m2025-12-21 11:25:50.330\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n", "Example: {'pubid': 27858166, 'question': 'Traumatic aortic injury: does the anatomy of the aortic arch influence aortic trauma severity?', 'context': {'contexts': ['Traumatic aortic injury (TAI) is a rare but life-threatening type of injury. We investigate whether the anatomy of the aortic arch influences the severity of aortic injury.', 'This is a retrospective study of twenty-two cases treated with TEVAR for TAI in our department from 2009 to 2014. Aortic injury was assessed in accordance with the recommendations of the Society of Vascular Surgery. We measured the aortic arch angle and the aortic arch index, based on the initial angio-CT scan, in each of the analyzed cases.', 'The mean aortic arch index and mean aortic arch angle were 6.8\\xa0cm and 58.3°, respectively, in the type I injury group; 4.4\\xa0cm and 45.9° in the type III group; 3.3\\xa0cm and 37° in the type IV group. There were substantial differences in both the aortic arch index and the aortic arch angle of the type III and IV groups. A multivariate analysis confirmed that the aortic arch angle was significantly associated with the occurrence of type III damage (OR 1.5; 95% CI 1.03-2.2).'], 'labels': ['PURPOSE', 'METHODS', 'RESULTS'], 'meshes': ['Adult', 'Aorta', 'Aorta, Thoracic', 'Endovascular Procedures', 'Female', 'Humans', 'Male', 'Middle Aged', 'Multivariate Analysis', 'Retrospective Studies', 'Stents', 'Thoracic Injuries', 'Tomography, X-Ray Computed', 'Trauma Severity Indices', 'Treatment Outcome', 'Young Adult'], 'reasoning_required_pred': ['y', 'e', 's'], 'reasoning_free_pred': ['y', 'e', 's']}, 'long_answer': 'The severity of TAI is influenced by the sharpness of the aortic arch. There is an inverse relationship between the severity of aortic injury and the aortic arch index.', 'final_decision': 'yes'}\n", "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n", "\u001b[32m2025-12-21 11:25:50.330\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n", "Example: {'pubid': 15223725, 'question': 'Does blood pressure change in treated hypertensive patients depending on whether it is measured by a physician or a nurse?', 'context': {'contexts': ['To determine whether there are differences between blood pressure (BP) measured by the nurse (NBP), BP measured by the physician (PBP) and self-measured BP in treated hypertensive patients and, if found, to evaluate their clinical importance.', 'An observational study is carried out with hypertensive patients recruited from two village-based community health centres in Catalonia (Spain) serving an area with a total population of 2800 inhabitants. All patients treated for hypertension visiting the health centre on a specific day of the week and during the same timetable between October 2000 and May 2001 were included.', 'The difference between physician-systolic BP and nurse-systolic BP was 5.16 mmHg (95% CI 2.62-7.7; p<0.001). The difference between physician-systolic BP and self-measured systolic BP was 4.67 mmHg (95% CI 0.89-8.44; p=0.016). The differences between nurse-systolic BP and self-measured systolic BP were not significant (0.49 mmHg; 95% CI 3.71-2.71; p=0.758). With regards to diastolic BP, no significant differences were found between the different ways of measurement. NBP gave the following values: sensitivity (Sn) of 92% and specificity (Sp) of 60%; positive predictive value (PPV) of 65.7% and negative predictive value (NPV) of 90% with a positive coefficient of probability (CP+) of 2.3 and a negative coefficient of probability (CP-) of 0.133. PBP gave the following results: Sn=72%; Sp=66.7%; PPV=64.3%; NPV=74.1%; CP+=2.16 and CP- = 0.420.'], 'labels': ['OBJECTIVES', 'METHOD', 'RESULTS'], 'meshes': ['Adult', 'Aged', 'Aged, 80 and over', 'Antihypertensive Agents', 'Blood Pressure Determination', 'Female', 'Humans', 'Hypertension', 'Male', 'Middle Aged', \"Nurse's Role\", \"Physician's Role\", 'Self Care', 'Spain'], 'reasoning_required_pred': ['m', 'a', 'y', 'b', 'e'], 'reasoning_free_pred': ['y', 'e', 's']}, 'long_answer': 'Systolic BP measured by the nurse in treated hypertensive patients is significantly lower than the readings obtained by the physician, and are almost identical to ambulatory BP monitoring. Blood pressure determination by the nurse is desirable not only for diagnosis but also to evaluate the level of control of blood pressure during the follow-up of treated hypertensive patients.', 'final_decision': 'yes'}\n", "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n", "\u001b[32m2025-12-21 11:25:50.343\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n", "Example: {'pubid': 24245816, 'question': 'Is trabecular bone related to primary stability of miniscrews?', 'context': {'contexts': ['To compare the primary stability of miniscrews inserted into bone blocks of different bone mineral densities (BMDs) with and without cortical bone, and investigate whether some trabecular properties could influence primary stability.', 'Fifty-two bone blocks were extracted from fresh bovine pelvic bone. Four groups were created based on bone type (iliac or pubic region) and presence or absence of cortical bone. Specimens were micro-computed tomography imaged to evaluate trabecular thickness, trabecular number, trabecular separation, bone volume density (BV/TV), BMD, and cortical thickness. Miniscrews 1.4 mm in diameter and 6 mm long were inserted into the bone blocks, and primary stability was evaluated by insertion torque (IT), mini-implant mobility (PTV), and pull-out strength (PS).', 'Intergroup comparison showed lower levels of primary stability when the BMD of trabecular bone was lower and in the absence of cortical bone (P≤.05). The Pearson correlation test showed correlation between trabecular number, trabecular thickness, BV/TV, trabecular BMD, total BMD, and IT, PTV, and PS. There was correlation between cortical thickness and IT and PS (P≤.05).'], 'labels': ['OBJECTIVE', 'MATERIALS AND METHODS', 'RESULTS'], 'meshes': ['Anatomy, Cross-Sectional', 'Animals', 'Bone Density', 'Bone Screws', 'Bone and Bones', 'Cattle', 'Dental Stress Analysis', 'Durapatite', 'Ilium', 'Image Processing, Computer-Assisted', 'Imaging, Three-Dimensional', 'Miniaturization', 'Orthodontic Anchorage Procedures', 'Orthodontic Appliances', 'Pubic Bone', 'Stress, Mechanical', 'Torque', 'X-Ray Microtomography'], 'reasoning_required_pred': ['y', 'e', 's'], 'reasoning_free_pred': ['y', 'e', 's']}, 'long_answer': 'Cancellous bone plays an important role in primary stability of mini-implants in the presence or absence of cortical bone.', 'final_decision': 'yes'}\n", "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 35%|███▌ | 175/500 [00:38<01:29, 3.61it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 35%|███▌ | 177/500 [00:39<01:52, 2.86it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 36%|███▌ | 178/500 [00:40<01:50, 2.91it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "\u001b[32m2025-12-21 11:25:52.958\u001b[0m | \u001b[31m\u001b[1mERROR \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 36%|███▌ | 179/500 [00:40<02:09, 2.48it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "\u001b[32m2025-12-21 11:25:52.987\u001b[0m | \u001b[31m\u001b[1mERROR \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 36%|███▌ | 181/500 [00:41<01:44, 3.05it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 37%|███▋ | 184/500 [00:42<01:28, 3.59it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 37%|███▋ | 186/500 [00:42<01:09, 4.55it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 37%|███▋ | 187/500 [00:42<01:16, 4.11it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "\u001b[32m2025-12-21 11:25:54.886\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n", "Example: {'pubid': 24433626, 'question': 'Prevalence of chronic conditions among Medicare Part A beneficiaries in 2008 and 2010: are Medicare beneficiaries getting sicker?', 'context': {'contexts': ['Medicare beneficiaries who have chronic conditions are responsible for a disproportionate share of Medicare fee-for-service expenditures. The objective of this study was to analyze the change in the health of Medicare beneficiaries enrolled in Part A (hospital insurance) between 2008 and 2010 by comparing the prevalence of 11 chronic conditions.', 'We conducted descriptive analyses using the 2008 and 2010 Chronic Conditions Public Use Files, which are newly available from the Centers for Medicare and Medicaid Services and have administrative (claims) data on 100% of the Medicare fee-for-service population. We examined the data by age, sex, and dual eligibility (eligibility for both Medicare and Medicaid).', \"Medicare Part A beneficiaries had more chronic conditions on average in 2010 than in 2008. The percentage increase in the average number of chronic conditions was larger for dual-eligible beneficiaries (2.8%) than for nondual-eligible beneficiaries (1.2%). The prevalence of some chronic conditions, such as congestive heart failure, ischemic heart disease, and stroke/transient ischemic attack, decreased. The deterioration of average health was due to other chronic conditions: chronic kidney disease, depression, diabetes, osteoporosis, rheumatoid arthritis/osteoarthritis. Trends in Alzheimer's disease, cancer, and chronic obstructive pulmonary disease showed differences by sex or dual eligibility or both.\"], 'labels': ['INTRODUCTION', 'METHODS', 'RESULTS'], 'meshes': ['Aged', 'Aged, 80 and over', 'Chronic Disease', 'Female', 'Health Services Research', 'Humans', 'Insurance Claim Review', 'Male', 'Medicare Part A', 'Middle Aged', 'Prevalence', 'Time Factors', 'United States'], 'reasoning_required_pred': ['y', 'e', 's'], 'reasoning_free_pred': ['m', 'a', 'y', 'b', 'e']}, 'long_answer': 'Analyzing the prevalence of 11 chronic conditions by using Medicare claims data provides a monitoring tool that can guide health care providers and policy makers in devising strategies to address chronic conditions and rising health care costs.', 'final_decision': 'yes'}\n", "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 18 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 38%|███▊ | 188/500 [00:42<01:15, 4.13it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m2025-12-21 11:25:54.896\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n", "Example: {'pubid': 9542484, 'question': 'Does successful completion of the Perinatal Education Programme result in improved obstetric practice?', 'context': {'contexts': ['To determine whether successful completion of the Perinatal Education Programme (PEP) improves obstetric practice.', \"The three midwife obstetric units (MOUs) in a health district of Mpumalanga were included in the study. Two MOUs enrolled in the PEP and the third did not. A 'before-and-after' study design was used to assess any changes in practice, and to monitor whether any changes occurred in the district during the time of the study; data were also collected at the third MOU. Data were collected by scoring of the obstetric files after the patient had delivered.\", 'We ascertained whether the obstetric history, syphilis testing, blood group testing, haemoglobin measurement and uterine growth assessment were performed during antenatal care along with whether appropriate action was taken. For intrapartum care, estimation of fetal weight, the performance of pelvimetry, blood pressure monitoring, urine testing, evaluation of head above pelvis, fetal heart rate monitoring, monitoring of contractions and plotting of cervical dilatation, and whether the appropriate actions were taken, were assessed.', 'Eight of the 13 midwives at the two MOUs completed the PEP and all demonstrated an improvement in knowledge. Case notes of 303 patients from the various clinics were studied. There was no change in the referral patterns of any of the clinics during the study period. The obstetric history was well documented, but in no group was there a satisfactory response to a detected problem; appropriate action was taken in between 0% and 12% of cases. Syphilis testing was performed in 56-82% of cases, with no difference between the groups. The haemoglobin level was measured in only 4-15% of patients, with no difference before or after completion of the PEP. Where a problem in uterine growth was detected, an appropriate response occurred in 0-8% of patients and no difference before or after completion of the PEP was ascertained. In all groups, estimation of fetal weight and pelvimetry were seldom performed, the urine and fetal heart rate documentation were moderately well done and the blood pressure monitoring, assessment of head above pelvis, monitoring of contractions and plotting of cervical dilatation were usually performed. No differences before or after the PEP were detected. Where problems were detected, appropriate actions taken during labour improved, but not significantly.'], 'labels': ['OBJECTIVE', 'METHOD', 'OUTCOME MEASURES', 'RESULTS'], 'meshes': ['Africa', 'Female', 'Humans', 'Midwifery', 'Perinatal Care', 'Pregnancy', 'Prenatal Care', 'Rural Population'], 'reasoning_required_pred': ['n', 'o'], 'reasoning_free_pred': ['n', 'o']}, 'long_answer': 'Completion of the obstetric manual of the PEP improved the knowledge of the midwives but no alteration in practice was detected.', 'final_decision': 'no'}\n", "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 18 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 38%|███▊ | 190/500 [00:43<00:55, 5.61it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "\u001b[32m2025-12-21 11:25:55.096\u001b[0m | \u001b[31m\u001b[1mERROR \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 17 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "\u001b[32m2025-12-21 11:25:55.145\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n", "Example: {'pubid': 23228527, 'question': 'Does music influence stress in mechanically ventilated patients?', 'context': {'contexts': ['Mechanically ventilated patients experience profound stress. Interventions are needed to ameliorate stress that does not cause adverse effects. The purpose of this study was to explore the influence of music on stress in a sample of patients over the duration of ventilatory support.RESEARCH METHODOLOGY/', 'Randomised controlled trial; randomised patients (56.8+16.9 years, 61% male, APACHE III 57.2+18.3) receiving ventilatory support to: (1) patient-directed music (PDM) where patients self-initiated music listening whenever desired from a preferred collection, (2) headphones only to block ICU noise, or (3) usual ICU care. Twenty-four hour urinary cortisol samples were collected from a sub-set of subjects with intact renal function and not receiving medications known to influence cortisol levels (n=65).', '12 ICUs in the Midwestern United States.', 'Urinary free cortisol (UFC), an integrative biomarker of stress.', 'Controlling for illness severity, gender, and baseline UFC (29-45 mg/day), mixed models analysis revealed no significant differences among groups in UFC over the course of ventilatory support.'], 'labels': ['OBJECTIVES', 'DESIGN', 'SETTING', 'MAIN OUTCOME MEASURES', 'RESULTS'], 'meshes': ['Adult', 'Aged', 'Critical Illness', 'Female', 'Humans', 'Hydrocortisone', 'Male', 'Middle Aged', 'Music Therapy', 'Respiration, Artificial', 'Stress, Psychological'], 'reasoning_required_pred': ['n', 'o'], 'reasoning_free_pred': ['n', 'o']}, 'long_answer': 'While music did not significantly reduce cortisol, less profound spikes in UFC levels were observed but that, given the limitations of the research, this observation could have occurred merely by chance.', 'final_decision': 'no'}\n", "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 18 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 38%|███▊ | 192/500 [00:43<00:41, 7.43it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m2025-12-21 11:25:56.175\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n", "Example: {'pubid': 15489384, 'question': 'Does reducing spasticity translate into functional benefit?', 'context': {'contexts': ['Spasticity and loss of function in an affected arm are common after stroke. Although botulinum toxin is used to reduce spasticity, its functional benefits are less easily demonstrated. This paper reports an exploratory meta-analysis to investigate the relationship between reduced arm spasticity and improved arm function.', 'Individual data from stroke patients in two randomised controlled trials of intra-muscular botulinum toxin were pooled. The Modified Ashworth Scale (elbow, wrist, fingers) was used to calculate a \"Composite Spasticity Index\". Data from the arm section of the Barthel Activities of Daily Living Index (dressing, grooming, and feeding) and three subjective measures (putting arm through sleeve, cleaning palm, cutting fingernails) were summed to give a \"Composite Functional Index\". Change scores and the time of maximum change were also calculated.', 'Maximum changes in both composite measures occurred concurrently in 47 patients. In 26 patients the improvement in spasticity preceded the improvement in function with 18 showing the reverse. There was a definite relationship between the maximum change in spasticity and the maximum change in arm function, independent of treatment (rho = -0.2822, p = 0.0008, n = 137). There was a clear relationship between the changes in spasticity and in arm function in patients treated with botulinum toxin (Dysport) at 500 or 1000 units (rho = -0.5679, p = 0.0090, n = 22; rho = -0.4430, p = 0.0018, n = 47), but not in those treated with placebo or 1500 units.'], 'labels': ['BACKGROUND', 'METHOD', 'RESULTS'], 'meshes': ['Activities of Daily Living', 'Aged', 'Arm', 'Botulinum Toxins, Type A', 'Double-Blind Method', 'Female', 'Humans', 'Injections, Intramuscular', 'Male', 'Middle Aged', 'Motor Activity', 'Muscle Spasticity', 'Neuromuscular Agents', 'Randomized Controlled Trials as Topic', 'Stroke', 'Stroke Rehabilitation'], 'reasoning_required_pred': ['y', 'e', 's'], 'reasoning_free_pred': ['y', 'e', 's']}, 'long_answer': 'Using a targeted meta-analytic approach, it is possible to demonstrate that reducing spasticity in the arm is associated with a significant improvement in arm function.', 'final_decision': 'yes'}\n", "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 16 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 39%|███▉ | 197/500 [01:00<09:04, 1.80s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 40%|███▉ | 199/500 [01:01<06:37, 1.32s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 40%|████ | 201/500 [01:01<04:59, 1.00s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 40%|████ | 202/500 [01:02<04:39, 1.07it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 42%|████▏ | 208/500 [01:02<01:45, 2.77it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 42%|████▏ | 210/500 [01:02<01:26, 3.36it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 42%|████▏ | 212/500 [01:03<01:25, 3.39it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 43%|████▎ | 215/500 [01:04<01:21, 3.51it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 44%|████▎ | 218/500 [01:04<01:00, 4.68it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 45%|████▌ | 225/500 [01:05<00:26, 10.57it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 45%|████▌ | 227/500 [01:05<00:31, 8.71it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 46%|████▌ | 229/500 [01:06<00:52, 5.15it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 46%|████▌ | 231/500 [01:06<00:48, 5.54it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 46%|████▋ | 232/500 [01:06<00:52, 5.10it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 47%|████▋ | 234/500 [01:07<00:50, 5.29it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 47%|████▋ | 236/500 [01:07<00:48, 5.46it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 47%|████▋ | 237/500 [01:07<00:54, 4.81it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 48%|████▊ | 239/500 [01:08<00:45, 5.75it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 48%|████▊ | 241/500 [01:08<00:48, 5.33it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 49%|████▉ | 245/500 [01:08<00:33, 7.64it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 50%|████▉ | 249/500 [01:09<00:25, 9.93it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 50%|█████ | 251/500 [01:09<00:37, 6.70it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 50%|█████ | 252/500 [01:10<00:41, 5.93it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 51%|█████ | 255/500 [01:10<00:32, 7.52it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 51%|█████ | 256/500 [01:10<00:37, 6.54it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 52%|█████▏ | 260/500 [01:11<00:32, 7.33it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 52%|█████▏ | 262/500 [01:11<00:31, 7.51it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 53%|█████▎ | 264/500 [01:11<00:31, 7.47it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 53%|█████▎ | 265/500 [01:12<00:46, 5.05it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 53%|█████▎ | 266/500 [01:12<00:50, 4.62it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 54%|█████▍ | 270/500 [01:12<00:39, 5.80it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 54%|█████▍ | 272/500 [01:12<00:29, 7.60it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 55%|█████▌ | 275/500 [01:13<00:24, 9.16it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 55%|█████▌ | 277/500 [01:13<00:28, 7.70it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 56%|█████▌ | 278/500 [01:13<00:40, 5.50it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 56%|█████▌ | 280/500 [01:14<00:35, 6.12it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 56%|█████▋ | 282/500 [01:14<00:32, 6.63it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 57%|█████▋ | 283/500 [01:14<00:44, 4.83it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 58%|█████▊ | 291/500 [01:15<00:27, 7.54it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 58%|█████▊ | 292/500 [01:16<00:36, 5.72it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 59%|█████▉ | 295/500 [01:16<00:31, 6.43it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 59%|█████▉ | 296/500 [01:17<00:43, 4.69it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 60%|█████▉ | 299/500 [01:17<00:36, 5.53it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 60%|██████ | 302/500 [01:17<00:23, 8.46it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 61%|██████ | 306/500 [01:18<00:22, 8.73it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 62%|██████▏ | 308/500 [01:19<00:41, 4.58it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 62%|██████▏ | 312/500 [01:19<00:30, 6.16it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 63%|██████▎ | 313/500 [01:19<00:30, 6.18it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 63%|██████▎ | 316/500 [01:19<00:24, 7.41it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 64%|██████▎ | 318/500 [01:20<00:19, 9.25it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 64%|██████▍ | 322/500 [01:20<00:16, 11.09it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 65%|██████▍ | 324/500 [01:20<00:17, 10.05it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 65%|██████▌ | 327/500 [01:21<00:33, 5.12it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 66%|██████▌ | 328/500 [01:22<00:40, 4.20it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 66%|██████▌ | 331/500 [01:22<00:34, 4.86it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 66%|██████▋ | 332/500 [01:22<00:35, 4.78it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 67%|██████▋ | 337/500 [01:23<00:18, 8.86it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 68%|██████▊ | 342/500 [01:23<00:16, 9.70it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 69%|██████▉ | 345/500 [01:24<00:30, 5.11it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 70%|███████ | 350/500 [01:25<00:16, 8.96it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 70%|███████ | 352/500 [01:25<00:14, 9.92it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 71%|███████ | 354/500 [01:25<00:17, 8.35it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 71%|███████ | 356/500 [01:25<00:16, 8.52it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 72%|███████▏ | 360/500 [01:37<02:53, 1.24s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 72%|███████▏ | 362/500 [01:37<02:03, 1.12it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "\u001b[32m2025-12-21 11:26:49.465\u001b[0m | \u001b[31m\u001b[1mERROR \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 73%|███████▎ | 365/500 [01:37<01:18, 1.73it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "\u001b[32m2025-12-21 11:26:49.743\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n", "Example: {'pubid': 17595200, 'question': 'Is there an intrauterine influence on obesity?', 'context': {'contexts': ['It has been suggested that increasing obesity levels in young women lead to intrauterine environments that, in turn, stimulate increased obesity among their offspring, generating an intergenerational acceleration of obesity levels. If this mechanism is important, the association of maternal body mass index (BMI) with offspring BMI should be stronger than the association of paternal with offspring BMI.', 'To compare the relative strengths of association of maternal and paternal BMI with offspring BMI at age 7.5, taking into account the possible effect of non-paternity.', 'We compared strength of association for maternal-offspring and paternal-offspring BMI for 4654 complete parent-offspring trios in the Avon Longitudinal Study of Parents and Children (ALSPAC), using unstandardised and standardised regression analysis. We carried out a sensitivity analysis to investigate the influence of non-paternity on these associations.', 'The strength of association between parental BMI and offspring BMI at age 7.5 was similar for both parents. Taking into account correlations between maternal and paternal BMI, performing standardised rather than unstandardised regression and carrying out a sensitivity analysis for non-paternity emphasised the robustness of the general similarity of the associations. The associations between high parental BMI (top decile) and offspring BMI are also similar for both parents.'], 'labels': ['BACKGROUND', 'OBJECTIVE', 'METHODS', 'RESULTS'], 'meshes': ['Adult', 'Body Mass Index', 'Child', 'Child, Preschool', 'Cross-Sectional Studies', 'Fathers', 'Female', 'Health Surveys', 'Humans', 'Longitudinal Studies', 'Male', 'Mothers', 'Obesity', 'Parent-Child Relations', 'Pregnancy', 'United Kingdom'], 'reasoning_required_pred': ['n', 'o'], 'reasoning_free_pred': ['n', 'o']}, 'long_answer': 'Comparison of mother-offspring and father-offspring associations for BMI suggests that intergenerational acceleration mechanisms do not make an important contribution to levels of childhood BMI within the population. Associations at later ages and for different components of body composition now require study.', 'final_decision': 'no'}\n", "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n", "\u001b[32m2025-12-21 11:26:49.760\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n", "Example: {'pubid': 15879722, 'question': \"Is cytokeratin immunoreactivity useful in the diagnosis of short-segment Barrett's oesophagus in Korea?\", 'context': {'contexts': [\"Cytokeratin 7/20 staining has been reported to be helpful in diagnosing Barrett's oesophagus and gastric intestinal metaplasia. However, this is still a matter of some controversy.\", \"To determine the diagnostic usefulness of cytokeratin 7/20 immunostaining for short-segment Barrett's oesophagus in Korea.\", \"In patients with Barrett's oesophagus, diagnosed endoscopically, at least two biopsy specimens were taken from just below the squamocolumnar junction. If goblet cells were found histologically with alcian blue staining, cytokeratin 7/20 immunohistochemical stains were performed. Intestinal metaplasia at the cardia was diagnosed whenever biopsy specimens taken from within 2 cm below the oesophagogastric junction revealed intestinal metaplasia. Barrett's cytokeratin 7/20 pattern was defined as cytokeratin 20 positivity in only the superficial gland, combined with cytokeratin 7 positivity in both the superficial and deep glands.\", \"Barrett's cytokeratin 7/20 pattern was observed in 28 out of 36 cases (77.8%) with short-segment Barrett's oesophagus, 11 out of 28 cases (39.3%) with intestinal metaplasia at the cardia, and nine out of 61 cases (14.8%) with gastric intestinal metaplasia. The sensitivity and specificity of Barrett's cytokeratin 7/20 pattern were 77.8 and 77.5%, respectively.\"], 'labels': ['BACKGROUND', 'OBJECTIVE', 'METHODS', 'RESULTS'], 'meshes': ['Adult', 'Aged', 'Barrett Esophagus', 'Biomarkers', 'Biopsy', 'Cardia', 'Esophagoscopy', 'Female', 'Humans', 'Intermediate Filament Proteins', 'Keratin-20', 'Keratin-7', 'Keratins', 'Male', 'Metaplasia', 'Middle Aged', 'Sensitivity and Specificity', 'Stomach'], 'reasoning_required_pred': ['y', 'e', 's'], 'reasoning_free_pred': ['y', 'e', 's']}, 'long_answer': \"Barrett's cytokeratin 7/20 pattern can be a useful marker for the diagnosis of short-segment Barrett's oesophagus, although the false positive or false negative rate is approximately 25%.\", 'final_decision': 'yes'}\n", "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n", "\u001b[32m2025-12-21 11:26:49.760\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n", "Example: {'pubid': 26085176, 'question': 'MR Diagnosis of Bone Metastases at 1.5 T and 3 T: Can STIR Imaging Be Omitted?', 'context': {'contexts': ['To date, no prospective comparative study of the diagnostic value of STIR versus T1-weighted (T1w) sequences at both 1.5 T and 3 T has been performed with special focus on the detectability of bone metastases.', '212 oncological patients had a whole-body MRI at 1.5 T and/or at 3 T. The standard protocol comprised STIR and T1w sequences. All patients who showed typical signs of bone metastases were included in the study. Evaluation of the images was performed by the calculation of the number of metastases by three independent readers and by visual assessment on a 4-point scale.', '86 patients fulfilled the inclusion criteria. The total number of metastases was significantly higher on T1w than on STIR images at both field strengths (p<0.05). T1w revealed a sensitivity of 99.72% (3 T) and 100.00% (1.5 T) versus STIR with 70.99 % (3 T) and 79.34 % (1.5 T). In 53% (38/72) of all patients, STIR detected fewer bone metastases in comparison with T1w at 3\\u200aT. At 1.5 T, STIR showed inferior results in 37.5 % (18/48) of all patients. Qualitative analysis indicated a significantly better lesion conspicuity, lesion delineation and an improved image quality on T1w compared to STIR imaging at both field strengths (p<0.05) with similar results for T1w at 1.5 T and 3 T, but inferior results for STIR especially at 3 T.'], 'labels': ['OBJECTIVE', 'MATERIALS AND METHODS', 'RESULTS'], 'meshes': ['Aged', 'Bone Neoplasms', 'Bone and Bones', 'Female', 'Humans', 'Image Enhancement', 'Magnetic Resonance Imaging', 'Male', 'Middle Aged', 'Prospective Studies', 'Sensitivity and Specificity', 'Spinal Neoplasms', 'Spine', 'Whole Body Imaging'], 'reasoning_required_pred': ['y', 'e', 's'], 'reasoning_free_pred': ['y', 'e', 's']}, 'long_answer': \"The whole-body MRI protocol for the detection of bone metastases could safely be limited to the T1w sequence in adults, especially at 3 T. There is no need for an additional STIR sequence. These initial results will have a major impact on the department's workflow if confirmed by larger studies as they will help reduce examination time and therefore save financial resources.\", 'final_decision': 'yes'}\n", "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m2025-12-21 11:26:49.778\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n", "Example: {'pubid': 27554179, 'question': 'Is routine dissection of the station 9 lymph nodes really necessary for primary lung cancer?', 'context': {'contexts': ['Mediastinal lymph node dissection is an essential component of lung cancer surgery; however, choosing mediastinal lymph nodes stations to be dissected is subjective. We carried out this research to investigate the need for dissection of station 9 lymph nodes during lung cancer surgery.', 'Patients with primary lung cancer who underwent radical surgery between 2010 and 2014 were retrospectively reviewed. Clinical, pathologic, and prognosis data were obtained and analyzed.', 'A total number of 1397 patients were included in this research. The metastasis rate of station 9 was 3.45%, which was significantly lower than other mediastinal stations. This metastasis rate was significantly correlated with pT stage, the lobe where the tumor was located, metastasis status of intrapulmonary lymph nodes, pTNM stage, and most of the other mediastinal lymph node stations. In males or ground glass opacity (GGO) patients, the metastasis of station 9 nodes was more unlikely to occur, even though there was no statistical significance. The staging results of most patients (99.63%) would not be impaired, even if station 9 nodes were not dissected, and the prognostic analysis showed that the metastasis status of station 9 had no significant influence on survival.'], 'labels': ['OBJECTIVES', 'METHODS', 'RESULTS'], 'meshes': ['Adult', 'Aged', 'Female', 'Humans', 'Lung Neoplasms', 'Lymph Node Excision', 'Lymphatic Metastasis', 'Male', 'Middle Aged', 'Prognosis', 'Retrospective Studies'], 'reasoning_required_pred': ['n', 'o'], 'reasoning_free_pred': ['n', 'o']}, 'long_answer': 'The metastasis rate of station 9 lymph nodes was significantly lower than other mediastinal stations in lung cancer patients. The metastasis status of station 9 had no significant influence on tumor staging or prognosis. Routine dissection of station 9 lymph nodes may not be necessary, especially in patients with a low T stage, upper or middle lobe tumors, or without intrapulmonary lymph node metastasis.', 'final_decision': 'no'}\n", "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 74%|███████▍ | 369/500 [01:37<00:45, 2.90it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m2025-12-21 11:26:49.785\u001b[0m | \u001b[31m\u001b[1mERROR \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "\u001b[32m2025-12-21 11:26:49.815\u001b[0m | \u001b[31m\u001b[1mERROR \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "\u001b[32m2025-12-21 11:26:49.835\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n", "Example: {'pubid': 28027677, 'question': 'Do prerecorded lecture VODcasts affect lecture attendance of first-yearpre-clinical Graduate Entry to Medicine students?', 'context': {'contexts': ['There is increasing concern amongst educators that the provision of recorded lectures may reduce student attendance of live lectures. We therefore sought to determine if the provision of prerecorded lecture video podcasts (VODcasts) to first-year Graduate Entry to Medicine (GEM) students, affected attendance at 21 Physiology lectures within three separate pre-clinical modules.', 'Data on lecture attendance, utilization of VODcasts, and whether VODcasts should replace live lectures were drawn from three surveys conducted in academic years 2014-2015 and 2015-2016 on all first-year GEM students in two first-year pre-clinical modules where prerecorded Physiology VODcasts were available for viewing or downloading prior to scheduled live lectures.', 'A total of 191/214 (89%) students responded to the three surveys, with 84.3% of students attending all 21 lectures in the study. Only 4% of students missed more than one lecture in each of the three lecture series, with 79% indicating that VODcasts should not replace lectures.'], 'labels': ['BACKGROUND', 'METHODS', 'RESULTS'], 'meshes': ['Adult', 'Education, Medical, Undergraduate', 'Female', 'Humans', 'Internet', 'Male', 'Surveys and Questionnaires', 'Teaching', 'Videotape Recording', 'Young Adult'], 'reasoning_required_pred': ['n', 'o'], 'reasoning_free_pred': ['n', 'o']}, 'long_answer': 'Therefore, we conclude that the attendance of pre-clinical GEM students at live lectures is not significantly impacted upon by the provision of lecture VODcasts, with most students viewing them as useful revision tools rather than as a replacement for live lectures.', 'final_decision': 'no'}\n", "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 75%|███████▍ | 373/500 [01:38<00:33, 3.84it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "\u001b[32m2025-12-21 11:26:50.516\u001b[0m | \u001b[31m\u001b[1mERROR \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 75%|███████▌ | 375/500 [01:39<00:40, 3.11it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 75%|███████▌ | 377/500 [01:40<00:46, 2.67it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 76%|███████▌ | 378/500 [01:40<00:46, 2.60it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 76%|███████▌ | 379/500 [01:42<01:04, 1.86it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 76%|███████▌ | 381/500 [01:42<00:48, 2.43it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "\u001b[32m2025-12-21 11:26:54.511\u001b[0m | \u001b[31m\u001b[1mERROR \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 18 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 77%|███████▋ | 383/500 [01:42<00:36, 3.21it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 77%|███████▋ | 386/500 [01:42<00:23, 4.79it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 78%|███████▊ | 388/500 [01:42<00:19, 5.87it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 78%|███████▊ | 389/500 [02:00<05:42, 3.09s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 79%|███████▊ | 393/500 [02:01<02:38, 1.48s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 79%|███████▉ | 395/500 [02:01<01:54, 1.09s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 80%|████████ | 400/500 [02:02<00:58, 1.71it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 80%|████████ | 402/500 [02:03<00:44, 2.21it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 81%|████████ | 404/500 [02:03<00:36, 2.66it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 81%|████████ | 405/500 [02:03<00:34, 2.76it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 81%|████████ | 406/500 [02:04<00:33, 2.80it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 82%|████████▏ | 408/500 [02:04<00:25, 3.64it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 82%|████████▏ | 409/500 [02:04<00:25, 3.63it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 82%|████████▏ | 410/500 [02:04<00:23, 3.79it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 82%|████████▏ | 412/500 [02:05<00:18, 4.72it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 83%|████████▎ | 416/500 [02:05<00:11, 7.12it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 84%|████████▍ | 421/500 [02:05<00:06, 11.61it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 85%|████████▍ | 423/500 [02:06<00:09, 8.01it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 85%|████████▌ | 425/500 [02:06<00:09, 7.71it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 86%|████████▌ | 428/500 [02:07<00:11, 6.03it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 87%|████████▋ | 433/500 [02:07<00:08, 7.50it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 87%|████████▋ | 434/500 [02:08<00:12, 5.18it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 88%|████████▊ | 438/500 [02:08<00:10, 5.64it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 88%|████████▊ | 440/500 [02:09<00:08, 7.08it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 89%|████████▊ | 443/500 [02:09<00:06, 8.66it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 89%|████████▉ | 445/500 [02:09<00:06, 8.27it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 89%|████████▉ | 447/500 [02:10<00:08, 6.24it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 90%|████████▉ | 448/500 [02:10<00:08, 5.94it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 90%|█████████ | 451/500 [02:10<00:06, 7.39it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 91%|█████████ | 455/500 [02:11<00:06, 7.30it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 91%|█████████▏| 457/500 [02:11<00:05, 7.41it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 92%|█████████▏| 458/500 [02:12<00:09, 4.37it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 92%|█████████▏| 461/500 [02:12<00:08, 4.66it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 93%|█████████▎| 464/500 [02:13<00:07, 4.64it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 94%|█████████▎| 468/500 [02:13<00:04, 6.46it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 94%|█████████▍| 470/500 [02:14<00:03, 8.13it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 94%|█████████▍| 472/500 [02:14<00:03, 8.27it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 95%|█████████▌| 476/500 [02:14<00:02, 9.87it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 96%|█████████▌| 479/500 [02:15<00:02, 7.16it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 96%|█████████▌| 480/500 [02:16<00:05, 3.94it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 97%|█████████▋| 483/500 [02:16<00:03, 4.85it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 98%|█████████▊| 488/500 [02:16<00:01, 8.94it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 98%|█████████▊| 492/500 [02:17<00:00, 9.31it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 99%|█████████▉| 494/500 [02:17<00:00, 9.10it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 99%|█████████▉| 496/500 [02:17<00:00, 6.96it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 100%|█████████▉| 498/500 [02:18<00:00, 6.10it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 100%|█████████▉| 499/500 [02:18<00:00, 4.14it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 100%|██████████| 500/500 [02:19<00:00, 3.59it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "Evaluation metrics: {'f1': 0.0, 'em': 0.0, 'acc': 0.655444607023838}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n" ] } ], "source": [ "# obtain SEWOptimizer after having more roles, default\n", "optimizer = SEWOptimizer(\n", " graph=sew_graph, \n", " evaluator=evaluator, \n", " llm=llm, \n", " max_steps=20,\n", " eval_rounds=3, \n", " repr_scheme=\"python\", \n", " optimize_mode=\"all\", \n", " order=\"zero-order\",\n", " max_rounds=20,\n", ")\n", "with suppress_logger_info():\n", " metrics = optimizer.evaluate(dataset=benchmark, eval_mode=\"test\")\n", "print(\"Evaluation metrics: \", metrics)" ] }, { "cell_type": "code", "execution_count": 27, "id": "8b05058e", "metadata": { "scrolled": true }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m2025-12-21 11:29:34.561\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m678\u001b[0m - \u001b[1mOptimizing the SEWWorkFlowGraph workflow with python representation.\u001b[0m\n", "\u001b[32m2025-12-21 11:29:34.562\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m682\u001b[0m - \u001b[1mRun initial evaluation on the original workflow ...\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 8%|▊ | 4/50 [00:02<00:19, 2.34it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 16%|█▌ | 8/50 [00:02<00:07, 5.35it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 22%|██▏ | 11/50 [00:02<00:05, 7.05it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 28%|██▊ | 14/50 [00:02<00:04, 8.54it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 36%|███▌ | 18/50 [00:03<00:03, 9.19it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 40%|████ | 20/50 [00:03<00:03, 8.68it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 46%|████▌ | 23/50 [00:04<00:05, 4.64it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 50%|█████ | 25/50 [00:05<00:06, 4.11it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 54%|█████▍ | 27/50 [00:05<00:04, 5.50it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 62%|██████▏ | 31/50 [00:05<00:02, 8.89it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 66%|██████▌ | 33/50 [00:05<00:01, 9.56it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 70%|███████ | 35/50 [00:06<00:01, 9.82it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 74%|███████▍ | 37/50 [00:06<00:01, 8.15it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 78%|███████▊ | 39/50 [00:06<00:01, 6.63it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 82%|████████▏ | 41/50 [00:06<00:01, 8.23it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 84%|████████▍ | 42/50 [00:07<00:01, 7.44it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 88%|████████▊ | 44/50 [00:07<00:01, 5.84it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 92%|█████████▏| 46/50 [00:07<00:00, 5.53it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 94%|█████████▍| 47/50 [00:08<00:00, 5.50it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 98%|█████████▊| 49/50 [00:08<00:00, 4.92it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 100%|██████████| 50/50 [00:09<00:00, 5.53it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "\u001b[32m2025-12-21 11:29:43.690\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m685\u001b[0m - \u001b[1mInitial metrics: {'f1': 0.0, 'em': 0.0, 'acc': 1.0}\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m2025-12-21 11:29:44.170\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.468 | Total tokens: 1725273 | Current cost: $0.000 | Current tokens: 67\u001b[0m\n", "\u001b[32m2025-12-21 11:29:44.779\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.468 | Total tokens: 1725400 | Current cost: $0.000 | Current tokens: 127\u001b[0m\n", "\u001b[32m2025-12-21 11:29:44.780\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m700\u001b[0m - \u001b[33m\u001b[1mError in step 0: can only concatenate str (not \"NoneType\") to str. Skip this step.\u001b[0m\n", "\u001b[32m2025-12-21 11:29:46.274\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.468 | Total tokens: 1725606 | Current cost: $0.000 | Current tokens: 206\u001b[0m\n", "\u001b[32m2025-12-21 11:29:46.750\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.468 | Total tokens: 1725875 | Current cost: $0.000 | Current tokens: 269\u001b[0m\n", "\u001b[32m2025-12-21 11:29:46.752\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m700\u001b[0m - \u001b[33m\u001b[1mError in step 1: can only concatenate str (not \"NoneType\") to str. Skip this step.\u001b[0m\n", "\u001b[32m2025-12-21 11:29:47.259\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.468 | Total tokens: 1725961 | Current cost: $0.000 | Current tokens: 86\u001b[0m\n", "\u001b[32m2025-12-21 11:29:47.878\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.468 | Total tokens: 1726096 | Current cost: $0.000 | Current tokens: 135\u001b[0m\n", "\u001b[32m2025-12-21 11:29:47.880\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m700\u001b[0m - \u001b[33m\u001b[1mError in step 2: can only concatenate str (not \"NoneType\") to str. Skip this step.\u001b[0m\n", "\u001b[32m2025-12-21 11:29:49.580\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.468 | Total tokens: 1726289 | Current cost: $0.000 | Current tokens: 193\u001b[0m\n", "\u001b[32m2025-12-21 11:29:50.209\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.468 | Total tokens: 1726547 | Current cost: $0.000 | Current tokens: 258\u001b[0m\n", "\u001b[32m2025-12-21 11:29:50.210\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m700\u001b[0m - \u001b[33m\u001b[1mError in step 3: can only concatenate str (not \"NoneType\") to str. Skip this step.\u001b[0m\n", "\u001b[32m2025-12-21 11:29:51.054\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.468 | Total tokens: 1726674 | Current cost: $0.000 | Current tokens: 127\u001b[0m\n", "\u001b[32m2025-12-21 11:29:51.729\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.468 | Total tokens: 1726865 | Current cost: $0.000 | Current tokens: 191\u001b[0m\n", "\u001b[32m2025-12-21 11:29:51.730\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36mparse_workflow_python_repr\u001b[0m:\u001b[36m403\u001b[0m - \u001b[33m\u001b[1mFailed to parse workflow string: 'llm_config'. Return the original workflow.\u001b[0m\n", "\u001b[32m2025-12-21 11:29:51.731\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m700\u001b[0m - \u001b[33m\u001b[1mError in step 4: can only concatenate str (not \"NoneType\") to str. Skip this step.\u001b[0m\n", "\u001b[32m2025-12-21 11:29:52.942\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.468 | Total tokens: 1726989 | Current cost: $0.000 | Current tokens: 124\u001b[0m\n", "\u001b[32m2025-12-21 11:29:53.529\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.469 | Total tokens: 1727181 | Current cost: $0.000 | Current tokens: 192\u001b[0m\n", "\u001b[32m2025-12-21 11:29:53.530\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m700\u001b[0m - \u001b[33m\u001b[1mError in step 5: can only concatenate str (not \"NoneType\") to str. Skip this step.\u001b[0m\n", "\u001b[32m2025-12-21 11:29:55.192\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.469 | Total tokens: 1727425 | Current cost: $0.000 | Current tokens: 244\u001b[0m\n", "\u001b[32m2025-12-21 11:29:55.836\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.469 | Total tokens: 1727732 | Current cost: $0.000 | Current tokens: 307\u001b[0m\n", "\u001b[32m2025-12-21 11:29:55.837\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m700\u001b[0m - \u001b[33m\u001b[1mError in step 6: can only concatenate str (not \"NoneType\") to str. Skip this step.\u001b[0m\n", "\u001b[32m2025-12-21 11:29:56.350\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.469 | Total tokens: 1727814 | Current cost: $0.000 | Current tokens: 82\u001b[0m\n", "\u001b[32m2025-12-21 11:29:56.921\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.469 | Total tokens: 1727955 | Current cost: $0.000 | Current tokens: 141\u001b[0m\n", "\u001b[32m2025-12-21 11:29:56.922\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m700\u001b[0m - \u001b[33m\u001b[1mError in step 7: can only concatenate str (not \"NoneType\") to str. Skip this step.\u001b[0m\n", "\u001b[32m2025-12-21 11:29:57.441\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.469 | Total tokens: 1728041 | Current cost: $0.000 | Current tokens: 86\u001b[0m\n", "\u001b[32m2025-12-21 11:29:58.135\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.469 | Total tokens: 1728176 | Current cost: $0.000 | Current tokens: 135\u001b[0m\n", "\u001b[32m2025-12-21 11:29:58.137\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m700\u001b[0m - \u001b[33m\u001b[1mError in step 8: can only concatenate str (not \"NoneType\") to str. Skip this step.\u001b[0m\n", "\u001b[32m2025-12-21 11:29:59.905\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.469 | Total tokens: 1728449 | Current cost: $0.000 | Current tokens: 273\u001b[0m\n", "\u001b[32m2025-12-21 11:30:00.396\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.469 | Total tokens: 1728776 | Current cost: $0.000 | Current tokens: 327\u001b[0m\n", "\u001b[32m2025-12-21 11:30:00.397\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m700\u001b[0m - \u001b[33m\u001b[1mError in step 9: can only concatenate str (not \"NoneType\") to str. Skip this step.\u001b[0m\n", "\u001b[32m2025-12-21 11:30:00.914\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.469 | Total tokens: 1728894 | Current cost: $0.000 | Current tokens: 118\u001b[0m\n", "\u001b[32m2025-12-21 11:30:01.504\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.469 | Total tokens: 1729036 | Current cost: $0.000 | Current tokens: 142\u001b[0m\n", "\u001b[32m2025-12-21 11:30:01.505\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m700\u001b[0m - \u001b[33m\u001b[1mError in step 10: can only concatenate str (not \"NoneType\") to str. Skip this step.\u001b[0m\n", "\u001b[32m2025-12-21 11:30:02.041\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.469 | Total tokens: 1729132 | Current cost: $0.000 | Current tokens: 96\u001b[0m\n", "\u001b[32m2025-12-21 11:30:02.630\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.469 | Total tokens: 1729273 | Current cost: $0.000 | Current tokens: 141\u001b[0m\n", "\u001b[32m2025-12-21 11:30:02.631\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m700\u001b[0m - \u001b[33m\u001b[1mError in step 11: can only concatenate str (not \"NoneType\") to str. Skip this step.\u001b[0m\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m2025-12-21 11:30:03.528\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.469 | Total tokens: 1729390 | Current cost: $0.000 | Current tokens: 117\u001b[0m\n", "\u001b[32m2025-12-21 11:30:04.135\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.469 | Total tokens: 1729575 | Current cost: $0.000 | Current tokens: 185\u001b[0m\n", "\u001b[32m2025-12-21 11:30:04.137\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m700\u001b[0m - \u001b[33m\u001b[1mError in step 12: can only concatenate str (not \"NoneType\") to str. Skip this step.\u001b[0m\n", "\u001b[32m2025-12-21 11:30:07.492\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.470 | Total tokens: 1730055 | Current cost: $0.000 | Current tokens: 480\u001b[0m\n", "\u001b[32m2025-12-21 11:30:08.127\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.470 | Total tokens: 1730602 | Current cost: $0.000 | Current tokens: 547\u001b[0m\n", "\u001b[32m2025-12-21 11:30:08.129\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36mparse_workflow_python_repr\u001b[0m:\u001b[36m403\u001b[0m - \u001b[33m\u001b[1mFailed to parse workflow string: 'llm_config'. Return the original workflow.\u001b[0m\n", "\u001b[32m2025-12-21 11:30:08.129\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m700\u001b[0m - \u001b[33m\u001b[1mError in step 13: can only concatenate str (not \"NoneType\") to str. Skip this step.\u001b[0m\n", "\u001b[32m2025-12-21 11:30:10.629\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.470 | Total tokens: 1730963 | Current cost: $0.000 | Current tokens: 361\u001b[0m\n", "\u001b[32m2025-12-21 11:30:11.143\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.470 | Total tokens: 1731391 | Current cost: $0.000 | Current tokens: 428\u001b[0m\n", "\u001b[32m2025-12-21 11:30:11.144\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36mparse_workflow_python_repr\u001b[0m:\u001b[36m403\u001b[0m - \u001b[33m\u001b[1mFailed to parse workflow string: 'llm_config'. Return the original workflow.\u001b[0m\n", "\u001b[32m2025-12-21 11:30:11.145\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m700\u001b[0m - \u001b[33m\u001b[1mError in step 14: can only concatenate str (not \"NoneType\") to str. Skip this step.\u001b[0m\n", "\u001b[32m2025-12-21 11:30:11.721\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.470 | Total tokens: 1731459 | Current cost: $0.000 | Current tokens: 68\u001b[0m\n", "\u001b[32m2025-12-21 11:30:12.310\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.470 | Total tokens: 1731598 | Current cost: $0.000 | Current tokens: 139\u001b[0m\n", "\u001b[32m2025-12-21 11:30:12.312\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m700\u001b[0m - \u001b[33m\u001b[1mError in step 15: can only concatenate str (not \"NoneType\") to str. Skip this step.\u001b[0m\n", "\u001b[32m2025-12-21 11:30:12.980\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.470 | Total tokens: 1731713 | Current cost: $0.000 | Current tokens: 115\u001b[0m\n", "\u001b[32m2025-12-21 11:30:13.976\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.470 | Total tokens: 1731925 | Current cost: $0.000 | Current tokens: 212\u001b[0m\n", "\u001b[32m2025-12-21 11:30:13.977\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36mparse_workflow_python_repr\u001b[0m:\u001b[36m403\u001b[0m - \u001b[33m\u001b[1mFailed to parse workflow string: 'llm_config'. Return the original workflow.\u001b[0m\n", "\u001b[32m2025-12-21 11:30:13.978\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m700\u001b[0m - \u001b[33m\u001b[1mError in step 16: can only concatenate str (not \"NoneType\") to str. Skip this step.\u001b[0m\n", "\u001b[32m2025-12-21 11:30:14.798\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.470 | Total tokens: 1732036 | Current cost: $0.000 | Current tokens: 111\u001b[0m\n", "\u001b[32m2025-12-21 11:30:15.826\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.470 | Total tokens: 1732279 | Current cost: $0.000 | Current tokens: 243\u001b[0m\n", "\u001b[32m2025-12-21 11:30:15.827\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36mparse_workflow_python_repr\u001b[0m:\u001b[36m403\u001b[0m - \u001b[33m\u001b[1mFailed to parse workflow string: 'llm_config'. Return the original workflow.\u001b[0m\n", "\u001b[32m2025-12-21 11:30:15.828\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m700\u001b[0m - \u001b[33m\u001b[1mError in step 17: can only concatenate str (not \"NoneType\") to str. Skip this step.\u001b[0m\n", "\u001b[32m2025-12-21 11:30:16.332\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.470 | Total tokens: 1732374 | Current cost: $0.000 | Current tokens: 95\u001b[0m\n", "\u001b[32m2025-12-21 11:30:16.933\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.470 | Total tokens: 1732510 | Current cost: $0.000 | Current tokens: 136\u001b[0m\n", "\u001b[32m2025-12-21 11:30:16.935\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m700\u001b[0m - \u001b[33m\u001b[1mError in step 18: can only concatenate str (not \"NoneType\") to str. Skip this step.\u001b[0m\n", "\u001b[32m2025-12-21 11:30:18.197\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.470 | Total tokens: 1732650 | Current cost: $0.000 | Current tokens: 140\u001b[0m\n", "\u001b[32m2025-12-21 11:30:18.948\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.models.model_utils\u001b[0m:\u001b[36mupdate_cost\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mTotal cost: $0.470 | Total tokens: 1732874 | Current cost: $0.000 | Current tokens: 224\u001b[0m\n", "\u001b[32m2025-12-21 11:30:18.949\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36mparse_workflow_python_repr\u001b[0m:\u001b[36m403\u001b[0m - \u001b[33m\u001b[1mFailed to parse workflow string: 'llm_config'. Return the original workflow.\u001b[0m\n", "\u001b[32m2025-12-21 11:30:18.949\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m700\u001b[0m - \u001b[33m\u001b[1mError in step 19: can only concatenate str (not \"NoneType\") to str. Skip this step.\u001b[0m\n", "\u001b[32m2025-12-21 11:30:18.950\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m707\u001b[0m - \u001b[1mReach the maximum number of steps 20. Stop the optimization.\u001b[0m\n", "\u001b[32m2025-12-21 11:30:18.951\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36moptimize\u001b[0m:\u001b[36m710\u001b[0m - \u001b[1mRestore the best graph from the snapshot ...\u001b[0m\n", "\u001b[32m2025-12-21 11:30:18.951\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.optimizers.sew_optimizer\u001b[0m:\u001b[36mrestore_best_graph\u001b[0m:\u001b[36m814\u001b[0m - \u001b[1mRestore the best graph from snapshot with metrics {'f1': 0.0, 'em': 0.0, 'acc': 1.0} ...\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347353.215337273)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347353.243786753)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347353.125685882)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347357.720076117)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347358.270733113)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347358.863155864)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347353.334136532)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347354.39975467)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347353.209169391)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347354.128592445)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347353.98250067)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347353.798837174)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347358.974167819)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347355.759404923)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347358.412640833)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347357.816352745)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347357.768596945)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347358.104714697)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347358.335749774)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347360.376488656)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347358.256406297)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347356.199108226)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347356.98623048)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347357.128828027)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347358.575323462)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347358.200521314)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347358.805413233)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347359.148978283)])']\n", "connector: \n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Unclosed connector\n", "connections: ['deque([(, 347359.029653025)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347361.151920167)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347359.801511558)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347357.745128563)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347357.9304153)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347355.864369012)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347356.383944296)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347356.766114234)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347360.98865574)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347361.780869577)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347360.598228732)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347361.503410503)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347360.372711264)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347361.329599313)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347365.246175697)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347360.877328605)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347360.779200241)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347363.64130032)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347362.927586678)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347362.628486384)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347361.554286214)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347362.42435867)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347362.676021152)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347362.265627124)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347359.996781839)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347363.121603973)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347364.464056635)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347362.585391407)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347363.41950657)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347363.418296831)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347363.653718933)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347365.722793379)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347363.710020375)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347364.625019794)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347363.892873317)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347365.094251841)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347365.332910201)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347365.696847318)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347364.803338913)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347362.110954163)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347366.619850639)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347365.810794477)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347366.253748032)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347366.694090471)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347365.442438299)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347366.695610944)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347366.095977217)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347366.850093483)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347367.040230949)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347366.334978589)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347367.296003452)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347368.574172014)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347369.734328719)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347368.435507657)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347365.370029668)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347368.271646671)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347367.96713635)])']\n", "connector: \n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347368.587221209)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347368.023178267)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347368.311555373)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347368.266128859)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347369.515704664)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347370.341347659)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347369.298166837)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347370.613344355)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347369.825287407)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347370.488181119)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347370.075478335)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347369.757739728)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347369.091398153)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347369.95416848)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347369.410630411)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347373.051306702)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347371.397327912)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347372.425049133)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347372.592326916)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347371.37368937)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347371.49594714)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347371.811619414)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347371.444700216)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347373.909826514)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347372.175550392)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347371.771365525)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347372.992888924)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347372.817027345)])']\n", "connector: \n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Unclosed connector\n", "connections: ['deque([(, 347372.798273209)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347375.069322596)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347373.941503569)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347374.21865955)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347374.37203208)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347376.05879385)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347373.913624622)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347375.409412865)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347374.551146838)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347374.843154412)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347372.618983946)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347372.988127956)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347376.242924746)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347373.187725058)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347372.860755278)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347375.83465402)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347376.991932662)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347375.782876487)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347378.28772522)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347378.083352689)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347376.706952609)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347377.357631229)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347377.000384438)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347377.436261906)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347375.807081741)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347375.171401058)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347375.745314584)])']\n", "connector: \n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Unclosed connector\n", "connections: ['deque([(, 347375.379483852)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347376.982732755)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347375.559984693)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347374.966264402)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347379.746738522)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347377.815918286)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347378.262233117)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347378.191867931)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347378.230641894)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347381.085801241)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347379.674406544)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347379.367776939)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347378.745908355)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347380.889477742)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347380.381209903)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347381.656204918)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347377.429749227)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347379.408376615)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347377.810470028)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347377.433640108)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347381.287050652)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347381.461631639)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347382.000118505)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347412.282346049)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347381.046632621)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347382.270863204)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347381.139285481)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347381.260165549)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347382.445210665)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Unclosed client session\n", "client_session: \n", "Evaluating workflow: 0%| | 0/500 [00:00, 347381.960623538)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347382.84258087)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347382.166983363)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347380.096320913)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347380.324833054)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347382.32235505)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347380.642101075)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347383.245484765)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347383.460065269)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347383.547080567)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347412.454170013)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347383.468067275)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347383.836238087)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347411.843138805)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347411.990069868)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347411.395923193)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347411.984676251)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347414.769953502)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347414.30894859)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347383.283191538)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347382.807484439)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347411.997900301)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347411.315761085)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347411.970046282)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347416.562048843)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347414.235726311)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347411.382453514)])']\n", "connector: \n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Unclosed connector\n", "connections: ['deque([(, 347414.287218839)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347418.241726148)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347418.861798159)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347413.66306146)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347418.024073612)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347413.685532878)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347416.409121152)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347415.952274926)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347414.121569343)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347416.684799266)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347418.108482368)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347418.30360918)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347416.433146801)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347414.105312592)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347420.233373374)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347414.127546116)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347418.081460075)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347419.521421681)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347419.036038039)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347418.087805606)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347418.89001409)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347419.54066707)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347419.805986987)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347419.952162665)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347420.185334622)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347419.832591658)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347418.827941282)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347419.444476593)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347416.428304115)])']\n", "connector: \n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Unclosed connector\n", "connections: ['deque([(, 347418.424162793)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347420.342242342)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347420.634558662)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347421.457986368)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347421.259229847)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347421.590878117)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347422.314639296)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347422.769342971)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347421.551415866)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347421.395239779)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347421.696558219)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347421.964267552)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347422.597080103)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347422.403447384)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347422.559809116)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347421.690963581)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347423.492631215)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347423.127565224)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347420.712847996)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347425.872020968)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347424.005973675)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347424.162915015)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347423.737008029)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347424.305043386)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347424.216462317)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347424.157755489)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347425.520879381)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347428.23105605)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Unclosed connector\n", "connections: ['deque([(, 347426.551815394)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347424.827531849)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347424.527720719)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347426.225277477)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347427.014903652)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347422.453484267)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347422.512511194)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347426.831844526)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347428.084143684)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347427.247812072)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347426.385532683)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347426.602177351)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347427.829135684)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347428.698020444)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347428.281746551)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347428.641405694)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347427.162930634)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347430.01507718)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347426.098715436)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347425.710600145)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347425.644037137)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347424.787181044)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347428.950777038)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347428.88378868)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347428.93150347)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347430.654249891)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347428.978759899)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347430.965765448)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347429.710014039)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347430.286572042)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347432.871941967)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347428.875476221)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347430.178155709)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347428.129931999)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347430.634579352)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347427.509095973)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347428.498580641)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347428.743675011)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347431.030501572)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347431.200414597)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347433.029633311)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347431.058711017)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347431.687043138)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347431.602339278)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347431.481975417)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347431.748103973)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347432.357201471)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347433.483793921)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347433.16381865)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347432.762289272)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347432.910511066)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347430.571520369)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347432.74461457)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347433.455783697)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347434.419640494)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347434.258414751)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347435.21096848)])']\n", "connector: \n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347435.621657582)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347434.230487363)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347435.264094801)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347435.524217367)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347436.101624999)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347433.991165273)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347432.898909707)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347435.384044944)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347435.861576018)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347433.124530698)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347434.561215042)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347436.162487933)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347435.084078734)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347436.254980544)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347436.870343884)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347439.761107084)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347436.268475007)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347438.518087094)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347437.512975362)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347437.023049284)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347437.473483486)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347438.561771992)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347437.887756608)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347436.017702956)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347435.680436462)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347435.111255791)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347435.574748113)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347439.14735885)])']\n", "connector: \n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347442.346681679)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347440.96139262)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347438.614883239)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347439.508570815)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347439.448046956)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347438.621956373)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347439.38815731)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347438.888361778)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347439.353369629)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347439.595026321)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347437.810907008)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347440.410678475)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347440.353908147)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347440.745858277)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347437.586254813)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347472.092673417)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347442.05274525)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347441.572299644)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347441.127102065)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347442.205284482)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347441.567137113)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347441.472110582)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347442.951041299)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347442.124991583)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347442.437104773)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347441.976186852)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347440.876738033)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347442.635991362)])']\n", "connector: \n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Unclosed connector\n", "connections: ['deque([(, 347442.552565518)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347442.819976527)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347443.735766473)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347472.223242844)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347474.36293907)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347443.359274455)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347474.811832487)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347474.410747071)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347471.465468927)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347474.342422671)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347472.019530216)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347472.179117271)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347443.297654102)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347474.586749142)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347442.697954813)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347472.253576221)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347443.187327257)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347475.208686748)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347471.800492822)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347471.806509922)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347475.299493633)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347472.237505098)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347474.290593227)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347472.379188035)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347472.018013693)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347471.928863835)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347478.723842321)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347477.398786593)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Unclosed connector\n", "connections: ['deque([(, 347476.71912413)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347477.63522709)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347474.380431678)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347476.753278099)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347479.065524712)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347476.370595085)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347478.894067255)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347484.297346185)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347474.30828076)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347474.367845894)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347474.237470882)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347476.646814592)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347478.93905771)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347478.501886895)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347479.041492592)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347478.727453145)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347480.616199766)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347479.408843859)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347479.337473384)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347479.516997901)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347483.149735734)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347478.960862631)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347478.232871293)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347479.553096322)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347481.642295922)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347476.727630537)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347477.344268513)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347481.213575061)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347480.69288894)])']\n", "connector: \n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347481.239118304)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347481.500949838)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347480.991084211)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347482.113141906)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347481.35568498)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347481.785115623)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347483.076657943)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347481.247323062)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347481.268534161)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347482.408299381)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347482.515145412)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347482.759046083)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347482.321236437)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347482.96303888)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347483.453872037)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347485.090399251)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347483.810538833)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347486.480692556)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347484.268415821)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347483.76903921)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347485.011692373)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347483.678278027)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347486.454787551)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347484.217306339)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347485.06293533)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347483.145534645)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347485.04219162)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347486.577225438)])']\n", "connector: \n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Unclosed connector\n", "connections: ['deque([(, 347482.567374103)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347486.765344783)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347486.556936667)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347487.33432428)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347486.948401492)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347487.024258174)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347486.388225609)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347486.879275046)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347488.038923007)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347489.447447426)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347485.06878774)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347485.218468866)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347484.920728017)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347486.112597856)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347488.438959795)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347488.284851353)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347488.715688514)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347490.591640136)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347489.468231018)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347491.063113273)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347490.175823237)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347489.485792092)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347489.555575458)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347489.80153554)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347490.212731942)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347489.831739414)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347489.372055627)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347491.082081742)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347487.757808771)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347488.150134887)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347493.137853391)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347490.375422026)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347491.446518849)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347490.922311673)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347491.148270742)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347491.150155035)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347491.710689101)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347493.357110132)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347491.880920175)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347493.774121423)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347491.572894453)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347494.498433256)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347493.089277454)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347492.344735111)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347494.215927399)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347491.076059238)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347501.279395852)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347503.622636476)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347504.382815537)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347504.277133327)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347505.462743997)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347504.899312051)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347504.293520186)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347504.274440423)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347492.780198706)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347492.743456217)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347503.785477818)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347504.270959528)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347503.809384261)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347504.21559076)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347503.324389577)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347504.876778014)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347504.214416014)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347503.058625384)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347503.745837372)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347503.563448004)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347504.070703483)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347504.470774668)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347504.757718762)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347505.778363383)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347506.70383694)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347506.595655556)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347506.063994344)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347506.794434879)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347506.741653484)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347506.615475803)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347507.415381506)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347506.556319294)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347507.060992526)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347506.255382037)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347506.660415406)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347506.998530319)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347507.084174282)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347508.637845407)])']\n", "connector: \n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347507.35981657)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347506.346662863)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347508.125240577)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347508.699599428)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347506.24543948)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347509.285542868)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347508.943082969)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347511.668769774)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347508.823809103)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347508.447066913)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347508.974076527)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347509.764866463)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347510.373686505)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347509.252555632)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347509.394154034)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347509.601913579)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347509.592279008)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347509.212035594)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347508.221649507)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347508.680806559)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347511.103486001)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347510.761968103)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347510.52593513)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347511.605037591)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347532.204704978)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347511.165270047)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347512.080478603)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347511.906100745)])']\n", "connector: \n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347511.967970243)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347511.705746188)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347532.379569773)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347510.776366358)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347532.360315929)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347512.241935635)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347511.908919924)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347509.585584645)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347531.737102815)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347531.716188654)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347532.448130734)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347534.722370952)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347534.424722666)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347532.665452943)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347532.354712556)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347532.146887264)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347533.278226072)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347536.914826221)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347512.560719854)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347537.694204213)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347537.879122331)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347511.91727111)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347511.956737799)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347532.090007627)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347532.095667115)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347532.384236748)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347535.905236441)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347536.108447451)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347536.788203487)])']\n", "connector: \n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347537.229938337)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347534.712855746)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347535.101565907)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347536.643444947)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347536.839163586)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347536.728700735)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347534.579971749)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347534.794113632)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347537.283108807)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347537.281849868)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347539.591742357)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347534.512215551)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347534.52447226)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347539.621414008)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347538.776262613)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347539.02803376)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347539.674811682)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347537.230995081)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347538.466240209)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347539.48859722)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347539.473751442)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347539.004217867)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347540.67755802)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347541.978155344)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347539.912166044)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347538.792035348)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347540.991981232)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347545.031342577)])']\n", "connector: \n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Unclosed connector\n", "connections: ['deque([(, 347540.468119743)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347541.690758277)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347541.174171578)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347541.764900615)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347541.320193682)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347541.994080306)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347542.310349414)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347543.079892682)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347541.69639452)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347542.168535125)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347541.83812425)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347542.022046023)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347542.879172432)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347543.054157265)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347542.978555653)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347544.278033356)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347541.173061411)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347544.667522898)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347544.38996167)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347543.84570003)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347543.708555324)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347544.027003227)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347544.194911987)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347543.915808743)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347545.398374801)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347545.072754995)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347544.011271972)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347544.719595401)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347545.540947615)])']\n", "connector: \n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347544.477728244)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347544.958978409)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347546.432072293)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347547.192348078)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347547.470950495)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347546.615679436)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347563.281622094)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347546.444710233)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347550.329634233)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347548.479386496)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347546.832913862)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347547.671127396)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347545.053795884)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347563.082380874)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347545.448274723)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347545.338299224)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347549.240983982)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347548.665210807)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347549.105661329)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347549.76126119)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347548.542565525)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347550.798979324)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347549.410654155)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347552.785020596)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347550.192346867)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347548.96136071)])']\n", "connector: \n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Unclosed connector\n", "connections: ['deque([(, 347548.932444674)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347548.895694219)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347549.429764518)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347562.47051294)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347549.864096375)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347547.899485985)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347551.136531061)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347552.32006479)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347551.573584502)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347552.190790274)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347551.208301222)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347551.360966672)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347551.760789279)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347562.591263416)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347551.622073495)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347552.014033462)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347553.385785373)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347553.369270441)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347553.541889353)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347562.408081574)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347551.225404721)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347571.148139539)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347562.255193245)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347562.48178361)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347562.774031445)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347564.770734642)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347562.447353722)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347562.147208387)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347562.253420338)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347553.613258608)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347564.477029492)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347564.087591126)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347563.077191315)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347562.715780763)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347564.242770526)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347564.684893891)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347563.671604167)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347564.440205056)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347564.70207937)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347566.347110646)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347566.431442635)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347566.542329536)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347564.598200321)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347566.899609135)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347566.010969004)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347566.276914425)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347567.007050796)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347565.657390613)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347566.310332088)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347593.33022113)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347567.199487379)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347567.316750507)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347569.016634026)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347567.412496376)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347567.599859218)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347567.512444477)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347569.350386524)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347568.412549563)])']\n", "connector: \n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347568.897764122)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347566.815185137)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347568.81912244)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347566.095497643)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347566.836944503)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347572.259571552)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347569.99217601)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347569.680837151)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347569.78612313)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347569.435336554)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347569.714903226)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347570.755492367)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347569.873277295)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347572.010465552)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347569.894716015)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347571.008735589)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347573.907129564)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347569.237460134)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347569.39758602)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347571.318639239)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347568.999186665)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347571.416443016)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347572.031272648)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347573.097112084)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347572.480494834)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347571.908730073)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347572.202940779)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347573.380029377)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347573.564682445)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347572.731891048)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347573.86475811)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347573.884841063)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347571.544403254)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347571.720678725)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347573.856287159)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347571.78543741)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347571.920056386)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347593.520657949)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347592.106723462)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347573.806243058)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347593.666737015)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347593.950467421)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347593.966627791)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347594.701056986)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347593.624261938)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347592.684892517)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347592.398093612)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347593.629942943)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347593.597385193)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347573.925177596)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347593.509104602)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347596.035665229)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347594.874539141)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347597.217750413)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347598.523232697)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347597.614662088)])']\n", "connector: \n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347596.649789363)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347595.96927719)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347598.382277823)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347595.998238032)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347597.628561012)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347596.723177023)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347598.424190482)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347599.651378757)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347597.065978248)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347600.899947849)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347599.911766052)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347597.216674546)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347597.053131032)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347597.460381341)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347598.984488312)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347597.408633997)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347597.356271268)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347598.200416266)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347599.11745898)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347599.670570357)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347601.534119882)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347601.200505365)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347601.510059041)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347600.954838063)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347600.939264773)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347602.964742929)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347601.090290105)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347602.081389742)])']\n", "connector: \n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347601.25828574)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347599.364945436)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347601.63536386)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347601.003100168)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347602.286499959)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347601.612789225)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347610.373739124)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347604.655527801)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347605.353091786)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347604.446831536)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347604.583519974)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347603.652563251)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347604.358517702)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347603.253061285)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347603.239145009)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347601.816683167)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347607.618120761)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347601.494174465)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347602.671673857)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347604.300264566)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347605.46709842)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347605.566633218)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347605.048427253)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347604.945735618)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347606.28125841)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347606.984139817)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347605.976450408)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347610.916024956)])']\n", "connector: \n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Unclosed connector\n", "connections: ['deque([(, 347606.541955203)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347606.012119496)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347606.777740141)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347607.04145222)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347608.605474381)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347606.994939311)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347604.142664979)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347604.492838184)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347607.801758483)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347608.588132549)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347608.863365644)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347608.417278683)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347609.746728125)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347610.4970287)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347608.294691993)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347609.261172871)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347608.954657346)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347609.897375662)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347607.177936039)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347607.8388688)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347609.354730703)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347609.595938692)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347607.541871692)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347606.856923646)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347611.944631359)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347610.620573108)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347610.812688582)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347611.602879231)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Unclosed connector\n", "connections: ['deque([(, 347610.778497227)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347612.263959647)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347612.072653677)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347611.931005896)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347612.491555601)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347612.061052525)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347612.322204275)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347612.586734994)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347614.016880603)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347610.537029716)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347609.755130599)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347610.136095177)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347613.897627336)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347622.845562817)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347623.314928744)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347622.954316625)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347613.775833237)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347623.119149876)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347613.850065788)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347613.342128032)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347623.757127735)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347622.231557349)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347613.791291972)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347622.732671582)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347622.913351257)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347622.359571931)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347623.129476617)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347626.19693314)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347624.71774557)])']\n", "connector: \n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347625.229790277)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347622.928518809)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347627.09215719)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347624.571397843)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347630.489083054)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347622.98062011)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347627.657019373)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347627.19700887)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347626.683434246)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347627.812923153)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347626.671679695)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347625.955867088)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347627.144951627)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347627.254236781)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347628.828761783)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347628.409710965)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347627.213174132)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347626.623171635)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347626.93569406)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347627.69140439)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347625.476011005)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347624.914470868)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347627.353315499)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347625.601704803)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347628.890754475)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347628.663792435)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347629.146211505)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347629.455042432)])']\n", "connector: \n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347629.833751735)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347628.978781684)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347629.990033161)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347629.249214934)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347630.366501346)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347629.86071495)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347629.60871889)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347630.317635984)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347629.763858123)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347630.709660379)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347630.174869173)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347633.429212937)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347628.659233186)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347632.357855111)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347630.841838325)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347631.170037909)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347632.895770176)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347631.431530913)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347631.931844881)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347631.883753714)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347632.63836493)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347632.830923654)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347632.072339318)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347633.542206604)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347630.789946261)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347632.208062805)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347632.876020138)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347632.74504495)])']\n", "connector: \n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347632.59611051)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347633.448211322)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347655.1993353)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347653.590051886)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347654.282389811)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347655.040801188)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347654.215127924)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347653.628596526)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347633.848970036)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347655.24028795)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347654.058476458)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347632.77340387)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347633.000769944)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347654.123665074)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347652.805679573)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347652.782412884)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347654.101252342)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347652.257210968)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347652.688384415)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347652.438162758)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347652.741338172)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347653.992027688)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347655.714123684)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347652.812151511)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347654.930922046)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347655.421170793)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347654.616355945)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347655.759770245)])']\n", "connector: \n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347658.296039833)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347657.667666603)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347658.209936217)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347659.195207332)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347654.817816066)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347657.848608905)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347658.399082775)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347657.98786298)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347657.003677024)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347658.346039574)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347658.285454529)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347680.052944558)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347678.102094909)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347678.256801687)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347678.194493348)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347678.482666365)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347678.187698833)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347678.004445787)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347677.600854101)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347675.445761214)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347657.306675083)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347656.68572477)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347677.747741414)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347679.21218243)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347677.684576459)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347678.172797531)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347678.308324821)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347679.594996416)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347677.98832042)])']\n", "connector: \n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Unclosed connector\n", "connections: ['deque([(, 347678.130103289)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347677.545553827)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347677.954203229)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347677.982237013)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347678.303046624)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347683.670772268)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347680.045319218)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347681.221716816)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347680.56766921)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347683.561124403)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347681.633073615)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347680.653750899)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347680.370617603)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347681.154616715)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347681.177158584)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347680.452865455)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347681.846123474)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347681.167780668)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347680.640570566)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347681.140590145)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347681.352571159)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347681.868100529)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347682.44281553)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347680.493658837)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347682.456351398)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347685.028399452)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347682.581424362)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347683.572360048)])']\n", "connector: \n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Unclosed connector\n", "connections: ['deque([(, 347686.00752614)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347683.484583273)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347682.591280311)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347683.238428092)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347685.51499753)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347696.865717028)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347684.365242518)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347683.813106581)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347684.188491569)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347683.881241379)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347683.820502741)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347684.066459532)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347684.990180714)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347684.978004577)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347684.726351741)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347686.520310249)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347686.37468047)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347686.011940432)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347685.921098683)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347686.144416601)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347687.346242893)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347686.491776441)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347686.543091139)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347684.31670966)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347688.232584114)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347684.551254372)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347688.29082127)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347687.168662549)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347687.742627701)])']\n", "connector: \n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347687.420244466)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347688.455673661)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347689.5139529)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347688.98596946)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347688.322636194)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347688.950009822)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347689.133768099)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347687.637760716)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347686.664200855)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347686.952489898)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347689.738093555)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347690.054730215)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347687.077563266)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347690.300920911)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347689.895608395)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347690.140817623)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347690.213881235)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347690.589060271)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347690.321742059)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347691.186677343)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347690.872032567)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347691.205666529)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347691.686179451)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347690.806717586)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347691.699329048)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347691.03006126)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347688.742566461)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347689.400180115)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Unclosed connector\n", "connections: ['deque([(, 347693.159619469)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347691.788679267)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347693.158530057)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347693.451258458)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347692.008892961)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347692.356191241)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347693.492497675)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347693.704923344)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347693.279950077)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347692.450585142)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347693.591799642)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347693.338612598)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347691.733849245)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347692.964350158)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347693.577287452)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347694.535507696)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347694.200317364)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347694.921651406)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347694.134789975)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347694.950013058)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347694.733651422)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347695.154145491)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347695.659263906)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347696.036115052)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347696.180905607)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347695.187999821)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347696.179783155)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347693.840275507)])']\n", "connector: \n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Unclosed connector\n", "connections: ['deque([(, 347696.52088552)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347695.772908885)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347696.070398677)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347696.895878886)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347699.38565346)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347697.128458603)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347699.383385712)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347698.096720074)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347697.097448341)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347697.968187254)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347697.413504675)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347697.656208675)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347697.894592634)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347697.849687871)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347696.285844153)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347696.202491993)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347698.279210517)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347698.278066138)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347698.722030606)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347698.748410997)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347699.20864768)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347699.412454592)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347713.838126915)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347699.968179284)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347700.646965669)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347713.110360118)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347700.016218599)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Unclosed connector\n", "connections: ['deque([(, 347699.763540104)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347713.840094159)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347698.363339982)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347698.528530062)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347698.821513545)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347698.562782383)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347714.376447744)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347712.399422258)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347713.635803222)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347713.8505138)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347713.106225434)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347712.904878378)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347713.824077624)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347713.11751007)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347713.043977513)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347713.672915878)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347713.800291277)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347714.327451449)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347712.886041725)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347715.795896063)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347713.139289663)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347715.483625834)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347700.520931638)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347718.165810184)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347717.825412144)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347718.554025569)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347717.666188821)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347736.740088314)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347717.522615378)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347717.430667296)])']\n", "connector: \n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347716.830667744)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347736.389997614)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347718.408635418)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347736.489071904)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347716.501808376)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347736.413796255)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347717.617746829)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347718.659130841)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347736.711205221)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347719.68938367)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347737.171746897)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347718.39997469)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347716.472487104)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347736.827710745)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347736.329537592)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347738.393618368)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347738.17994517)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347738.168663279)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347740.220990573)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347738.055445355)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347738.059390551)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347718.610163744)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347737.843271365)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347738.973344643)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347740.601918095)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347738.060493218)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347741.034966588)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347738.877801245)])']\n", "connector: \n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Unclosed connector\n", "connections: ['deque([(, 347739.808014804)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347740.478804788)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347740.246987936)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347740.678817735)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347739.862562726)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347738.024247144)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347739.957360768)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347741.078138312)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347737.802675619)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347740.557460905)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347753.325736548)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347741.88087749)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347742.136420128)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347742.659688653)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347744.186195313)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347744.132617926)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347740.418405283)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347740.600824992)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347742.445963412)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347740.573087456)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347742.167744763)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347740.388334678)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347740.870819528)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347744.154104782)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347743.04872123)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347743.711693749)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347744.338546714)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347743.798617736)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Unclosed connector\n", "connections: ['deque([(, 347745.650434768)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347744.677373336)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347744.453724838)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347744.713345374)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347744.673735061)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347744.981148047)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347744.808736483)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347745.327616184)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347743.178803236)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347745.649335556)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347742.809385824)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347743.455820273)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347743.500689263)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347746.501323533)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347746.414889006)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347745.585680063)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347745.844986425)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347747.124366271)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347746.192220742)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347748.744644471)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347746.607380491)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347746.716640819)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347746.092956415)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347747.870276053)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347747.564922056)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347748.391146005)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347746.914863372)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347746.856861103)])']\n", "connector: \n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Unclosed connector\n", "connections: ['deque([(, 347748.505177249)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347751.19869564)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347748.54317188)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347750.488902701)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347750.03680137)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347748.91867291)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347749.536072486)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347749.787565958)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347749.104489454)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347749.878363828)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347748.599720315)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347748.451841445)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347747.894104041)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347749.557225938)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347748.203799305)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347747.841107855)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347750.576066546)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347751.36832043)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347750.525124936)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347751.206872968)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347751.911436209)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347751.252484808)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347752.021797883)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347754.648081642)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347752.178614718)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347751.813509382)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347752.915481309)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347752.636077653)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347753.131105109)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347750.577199642)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347750.552719208)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347754.590556128)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347753.461416152)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347753.208062385)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347753.530512579)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347755.898092952)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347753.66177725)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347754.832456472)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347753.16874289)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347754.075681281)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347755.134705515)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347754.576536079)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347755.409464353)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347755.379025731)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347752.66075953)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347753.071231368)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347755.616534611)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347755.501200424)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347757.059464497)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347756.172998408)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347755.573147153)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347755.933411157)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347756.211299343)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347756.102138497)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347755.818985612)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347755.671985255)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Unclosed connector\n", "connections: ['deque([(, 347758.678827536)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347757.201248474)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347758.629273908)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347754.974488462)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347759.268293141)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347757.609724942)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347759.073748979)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347759.066853304)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347757.97852651)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347758.137852477)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347758.364356933)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347758.452671695)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347758.357245928)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347758.802966807)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347759.06092185)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347760.51664174)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347759.217046365)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347760.0994647)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347758.569635197)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347760.675519486)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347760.52845275)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347757.701040267)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347760.619396481)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347772.735398484)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347761.322776404)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347761.83712155)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347760.472148606)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347761.205303804)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347760.763078591)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347773.782312002)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347773.189021375)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347772.858064713)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347760.52369087)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347760.289397458)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347760.851323789)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347761.100549403)])']\n", "connector: \n", "Unclosed connector\n", "connections: ['deque([(, 347773.268064767)])']\n", "connector: \n", "Unclosed client session\n", "client_session: \n", "Unclosed connector\n", "connections: ['deque([(, 347772.99881351)])']\n", "connector: \n", "Evaluating workflow: 1%| | 3/500 [00:01<04:10, 1.99it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 1%| | 5/500 [00:02<02:32, 3.25it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 1%|▏ | 7/500 [00:02<01:56, 4.23it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 2%|▏ | 9/500 [00:02<01:22, 5.98it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 3%|▎ | 13/500 [00:02<00:51, 9.47it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 3%|▎ | 15/500 [00:02<00:43, 11.24it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 4%|▍ | 20/500 [00:03<00:50, 9.54it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 5%|▍ | 24/500 [00:04<01:10, 6.79it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 5%|▌ | 26/500 [00:04<01:10, 6.76it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 6%|▌ | 28/500 [00:05<01:09, 6.82it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 6%|▌ | 30/500 [00:05<01:14, 6.28it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 7%|▋ | 33/500 [00:05<00:57, 8.10it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 8%|▊ | 38/500 [00:06<00:54, 8.40it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 8%|▊ | 40/500 [00:06<01:01, 7.45it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 8%|▊ | 42/500 [00:07<01:14, 6.18it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 9%|▉ | 44/500 [00:07<00:57, 7.95it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 9%|▉ | 46/500 [00:07<01:29, 5.05it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 10%|▉ | 49/500 [00:08<01:11, 6.28it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 10%|█ | 51/500 [00:08<00:55, 8.13it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 11%|█ | 55/500 [00:08<00:43, 10.31it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 12%|█▏ | 58/500 [00:08<00:34, 12.92it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 12%|█▏ | 60/500 [00:09<00:53, 8.26it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 12%|█▏ | 62/500 [00:09<01:00, 7.28it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 13%|█▎ | 63/500 [00:10<01:35, 4.56it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 13%|█▎ | 66/500 [00:10<01:08, 6.32it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 13%|█▎ | 67/500 [00:10<01:12, 5.94it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 14%|█▍ | 71/500 [00:11<00:57, 7.43it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 15%|█▍ | 73/500 [00:11<00:59, 7.19it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 15%|█▌ | 75/500 [00:11<00:45, 9.36it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 16%|█▌ | 79/500 [00:12<00:49, 8.53it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 16%|█▌ | 81/500 [00:12<01:01, 6.79it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 17%|█▋ | 83/500 [00:12<01:01, 6.81it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 17%|█▋ | 85/500 [00:13<00:47, 8.77it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 17%|█▋ | 87/500 [00:13<00:56, 7.25it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 18%|█▊ | 90/500 [00:14<01:10, 5.79it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 18%|█▊ | 92/500 [00:14<01:01, 6.60it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 19%|█▉ | 94/500 [00:14<00:57, 7.08it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 20%|█▉ | 98/500 [00:15<00:53, 7.56it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 20%|██ | 102/500 [00:15<00:48, 8.18it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 21%|██ | 105/500 [00:16<00:49, 7.92it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 21%|██ | 106/500 [00:16<01:04, 6.13it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 22%|██▏ | 108/500 [00:16<01:01, 6.35it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 22%|██▏ | 111/500 [00:17<00:56, 6.90it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 23%|██▎ | 114/500 [00:17<00:54, 7.13it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 23%|██▎ | 115/500 [00:17<00:50, 7.57it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 23%|██▎ | 117/500 [00:17<00:48, 7.93it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 24%|██▎ | 118/500 [00:18<01:08, 5.59it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 24%|██▍ | 121/500 [00:18<00:50, 7.51it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 25%|██▍ | 124/500 [00:18<00:53, 6.98it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 25%|██▌ | 126/500 [00:19<00:48, 7.69it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 25%|██▌ | 127/500 [00:19<00:49, 7.49it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 26%|██▌ | 128/500 [00:19<00:57, 6.43it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 26%|██▋ | 132/500 [00:19<00:51, 7.10it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 27%|██▋ | 135/500 [00:20<00:55, 6.60it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 27%|██▋ | 137/500 [00:20<00:58, 6.17it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 28%|██▊ | 140/500 [00:21<00:43, 8.33it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 29%|██▊ | 143/500 [00:21<00:35, 10.09it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 29%|██▉ | 145/500 [00:21<00:29, 11.88it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 29%|██▉ | 147/500 [00:21<00:36, 9.72it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 30%|███ | 150/500 [00:22<00:31, 10.95it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 30%|███ | 152/500 [00:22<01:10, 4.94it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 31%|███ | 154/500 [00:23<01:03, 5.45it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 31%|███▏ | 157/500 [00:23<00:58, 5.90it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 32%|███▏ | 159/500 [00:24<01:06, 5.15it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 33%|███▎ | 163/500 [00:24<00:37, 8.97it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 33%|███▎ | 165/500 [00:24<00:36, 9.18it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 33%|███▎ | 167/500 [00:24<00:38, 8.57it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 34%|███▍ | 170/500 [00:25<00:40, 8.12it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 34%|███▍ | 171/500 [00:25<00:55, 5.90it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 34%|███▍ | 172/500 [00:26<01:07, 4.89it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 35%|███▍ | 174/500 [00:26<00:53, 6.07it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 35%|███▌ | 175/500 [00:26<01:04, 5.07it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metricsmetrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", " {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "\u001b[32m2025-12-21 11:30:55.598\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n", "Example: {'pubid': 25156467, 'question': 'Should early extubation be the goal for children after congenital cardiac surgery?', 'context': {'contexts': ['We sought to determine the feasibility and assess the clinical outcomes associated with an early extubation strategy for all children undergoing congenital heart surgery, including neonates (age,<30 days).', 'We performed a linked database analysis of all patients undergoing congenital heart surgery from July 1, 2010 to December 31, 2012. We collected data on the cardiac diagnoses, preoperative status, procedure, and postoperative course, including the duration of invasive and noninvasive ventilation, failure of extubation, hemodynamic data, length of stay, complications, and mortality. A multivariable model was used to assess the independent factors associated with an inability to extubate within the operating room and with delayed extubation (>24 hours).', 'We operated on 613 children, including 97 neonates. Intraoperative extubation was achieved in 71% of the cases and early extubation (≤ 24 hours) was achieved in 89% of the cases. The overall mortality was 1.5% (9 of 613 patients). Early extubation was associated with lower mortality (1% vs 9%, P<.001) and a lower rate of reintubation (4% vs 23%, P<.001) compared with delayed extubation. Notably, 63% of the neonates were extubated within 24 hours, including 67% of arterial switch operations and 54% of total anomalous pulmonary venous return repairs. Norwood operations were the only procedure in which no patient was extubated within the first 24 hours. Multivariable logistic regression demonstrated that the predictors of delayed extubation included preoperative mechanical ventilation, weight<5 kg, a longer procedure time, and the need for postoperative inotrope support. Implementation of an early extubation strategy was associated with low rates of complications (5.1 per 10 procedures), short lengths of intensive care unit stay (median, 1 day; interquartile range, 1-3), and short hospital stays (median, 4 days; interquartile range, 3-6).'], 'labels': ['OBJECTIVE', 'METHODS', 'RESULTS'], 'meshes': ['Age Factors', 'Airway Extubation', 'Cardiac Surgical Procedures', 'Feasibility Studies', 'Female', 'Heart Defects, Congenital', 'Humans', 'Infant', 'Infant, Newborn', 'Intensive Care Units, Pediatric', 'Length of Stay', 'Logistic Models', 'Male', 'Multivariate Analysis', 'Odds Ratio', 'Retrospective Studies', 'Risk Factors', 'Time Factors', 'Treatment Outcome'], 'reasoning_required_pred': ['m', 'a', 'y', 'b', 'e'], 'reasoning_free_pred': ['y', 'e', 's']}, 'long_answer': 'Most children undergoing congenital heart surgery can be extubated in the operating room. Most neonates, including many undergoing complex procedures, can be extubated within the first 24 hours after surgery. Early extubation was associated with low morbidity rates and short lengths of intensive care unit and hospital stays.', 'final_decision': 'yes'}\n", "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 5 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 36%|███▌ | 178/500 [00:35<08:04, 1.50s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "\u001b[32m2025-12-21 11:30:58.251\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n", "Example: {'pubid': 15588538, 'question': 'Chronic functional somatic symptoms: a single syndrome?', 'context': {'contexts': ['Reliable longitudinal data of patients with functional somatic symptoms in general practice are lacking.', 'To identify distinctive features in patients with chronic functional somatic symptoms, and to determine whether these symptoms support the hypothesis of the existence of specific somatic syndromes.', 'Observational study, with a comparison control group.', 'Four primary care practices affiliated with the University of Nijmegen in the Netherlands.', 'One hundred and eighty-two patients diagnosed between 1998 and 2002 as having chronic functional somatic symptoms and 182 controls matched by age, sex, socioeconomic status, and practice were included. Data on comorbidity, referrals, diagnostic tests, and hospital admissions over a period of 10 years prior to the diagnosis were collected. Medication use and number of visits to the general practitioner (GP) were extracted from the moment computerised registration was started.', 'In the 10 years before the diagnosis of chronic functional somatic symptoms, significantly more patients than controls presented functional somatic symptoms in at least two body systems, and used more somatic and psychotropic drugs. They visited the GP twice as much, statistically had significantly more psychiatric morbidity, and were referred more often to mental health workers and somatic specialists. The number of patients undergoing diagnostic tests was higher for patients with chronic functional somatic symptoms than for controls, but hospital admissions rates were equal.'], 'labels': ['BACKGROUND', 'AIMS', 'DESIGN OF STUDY', 'SETTING', 'METHOD', 'RESULTS'], 'meshes': ['Adolescent', 'Adult', 'Aged', 'Aged, 80 and over', 'Case-Control Studies', 'Child', 'Chronic Disease', 'Female', 'Hospitalization', 'Humans', 'Male', 'Middle Aged', 'Netherlands', 'Patient Acceptance of Health Care', 'Referral and Consultation', 'Somatoform Disorders', 'Syndrome'], 'reasoning_required_pred': ['n', 'o'], 'reasoning_free_pred': ['n', 'o']}, 'long_answer': 'Patients with chronic functional somatic symptoms have a great diversity of functional somatic symptoms. They use more somatic and psychotropic drugs than controls in the years before diagnosis. Moreover, they show high rates of referrals and psychiatric morbidity. The diversity of symptoms of patients with chronic functional somatic symptoms supports the concept that symptoms do not cluster in well defined distinct syndromes. Therefore, patients with chronic functional somatic symptoms should preferably not be classified into medical subspecialty syndromes.', 'final_decision': 'no'}\n", "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 36%|███▌ | 179/500 [00:37<08:50, 1.65s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m2025-12-21 11:30:58.455\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n", "Example: {'pubid': 24433626, 'question': 'Prevalence of chronic conditions among Medicare Part A beneficiaries in 2008 and 2010: are Medicare beneficiaries getting sicker?', 'context': {'contexts': ['Medicare beneficiaries who have chronic conditions are responsible for a disproportionate share of Medicare fee-for-service expenditures. The objective of this study was to analyze the change in the health of Medicare beneficiaries enrolled in Part A (hospital insurance) between 2008 and 2010 by comparing the prevalence of 11 chronic conditions.', 'We conducted descriptive analyses using the 2008 and 2010 Chronic Conditions Public Use Files, which are newly available from the Centers for Medicare and Medicaid Services and have administrative (claims) data on 100% of the Medicare fee-for-service population. We examined the data by age, sex, and dual eligibility (eligibility for both Medicare and Medicaid).', \"Medicare Part A beneficiaries had more chronic conditions on average in 2010 than in 2008. The percentage increase in the average number of chronic conditions was larger for dual-eligible beneficiaries (2.8%) than for nondual-eligible beneficiaries (1.2%). The prevalence of some chronic conditions, such as congestive heart failure, ischemic heart disease, and stroke/transient ischemic attack, decreased. The deterioration of average health was due to other chronic conditions: chronic kidney disease, depression, diabetes, osteoporosis, rheumatoid arthritis/osteoarthritis. Trends in Alzheimer's disease, cancer, and chronic obstructive pulmonary disease showed differences by sex or dual eligibility or both.\"], 'labels': ['INTRODUCTION', 'METHODS', 'RESULTS'], 'meshes': ['Aged', 'Aged, 80 and over', 'Chronic Disease', 'Female', 'Health Services Research', 'Humans', 'Insurance Claim Review', 'Male', 'Medicare Part A', 'Middle Aged', 'Prevalence', 'Time Factors', 'United States'], 'reasoning_required_pred': ['y', 'e', 's'], 'reasoning_free_pred': ['m', 'a', 'y', 'b', 'e']}, 'long_answer': 'Analyzing the prevalence of 11 chronic conditions by using Medicare claims data provides a monitoring tool that can guide health care providers and policy makers in devising strategies to address chronic conditions and rising health care costs.', 'final_decision': 'yes'}\n", "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 2 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 36%|███▌ | 180/500 [00:38<06:52, 1.29s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m2025-12-21 11:30:58.595\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n", "Example: {'pubid': 16155169, 'question': 'Percutaneous ethanol injection for benign cystic thyroid nodules: is aspiration of ethanol-mixed fluid advantageous?', 'context': {'contexts': ['We evaluated the differences between percutaneous ethanol injection with and without aspiration of ethanol-mixed fluid for treatment of benign cystic thyroid nodules.', 'We examined 60 patients with benign cystic thyroid nodules confirmed by fine-needle aspiration biopsy and divided them into 2 groups according to nonaspiration (group A, n = 30) or aspiration (group B, n = 30) of ethanol-mixed fluid after intracystic ethanol injection. We evaluated in both groups the complete disappearance of the cystic portion of the thyroid nodule on follow-up ultrasonography (first follow-up ultrasonography; mean, 4.6 months in group A; mean, 4.4 months in group B) (chi-square test), side effects or complications during and after the procedure (chi-square test), and the total procedure time (Student t test).', 'Most patients showed complete disappearance of the cystic portion of the thyroid nodule (group A, n = 29; group B, n = 28), and they revealed no recurrence on follow-up ultrasonography. There was no statistical difference in the success rates between group A and group B (P>.05). Pain, the most common side effect, and other mild side effects or complications occurred in small numbers of patients in each group, but there was no significant difference in side effects or complications between the 2 groups (P>.05), except for intracystic hemorrhage (P<.05) and the complaint of all group B patients due to a double puncture (P<.001). The total procedure time was nearly double in group B than in group A because of the additional procedures, such as complete evacuation of the ethanol-mixed fluid and the 10-minute compression.'], 'labels': ['BACKGROUND AND PURPOSE', 'METHODS', 'RESULTS'], 'meshes': ['Adult', 'Aged', 'Cysts', 'Ethanol', 'Female', 'Humans', 'Injections, Intralesional', 'Male', 'Middle Aged', 'Sclerotherapy', 'Suction', 'Thyroid Nodule', 'Treatment Outcome', 'Ultrasonography'], 'reasoning_required_pred': ['n', 'o'], 'reasoning_free_pred': ['n', 'o']}, 'long_answer': 'Percutaneous ethanol injection without aspiration of ethanol-mixed fluid seems to be the preferable method of treatment of benign cystic thyroid nodules from the perspective of both the physician and the patient.', 'final_decision': 'no'}\n", "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 2 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 36%|███▌ | 181/500 [00:38<05:14, 1.01it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m2025-12-21 11:30:58.606\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n", "Example: {'pubid': 15489384, 'question': 'Does reducing spasticity translate into functional benefit?', 'context': {'contexts': ['Spasticity and loss of function in an affected arm are common after stroke. Although botulinum toxin is used to reduce spasticity, its functional benefits are less easily demonstrated. This paper reports an exploratory meta-analysis to investigate the relationship between reduced arm spasticity and improved arm function.', 'Individual data from stroke patients in two randomised controlled trials of intra-muscular botulinum toxin were pooled. The Modified Ashworth Scale (elbow, wrist, fingers) was used to calculate a \"Composite Spasticity Index\". Data from the arm section of the Barthel Activities of Daily Living Index (dressing, grooming, and feeding) and three subjective measures (putting arm through sleeve, cleaning palm, cutting fingernails) were summed to give a \"Composite Functional Index\". Change scores and the time of maximum change were also calculated.', 'Maximum changes in both composite measures occurred concurrently in 47 patients. In 26 patients the improvement in spasticity preceded the improvement in function with 18 showing the reverse. There was a definite relationship between the maximum change in spasticity and the maximum change in arm function, independent of treatment (rho = -0.2822, p = 0.0008, n = 137). There was a clear relationship between the changes in spasticity and in arm function in patients treated with botulinum toxin (Dysport) at 500 or 1000 units (rho = -0.5679, p = 0.0090, n = 22; rho = -0.4430, p = 0.0018, n = 47), but not in those treated with placebo or 1500 units.'], 'labels': ['BACKGROUND', 'METHOD', 'RESULTS'], 'meshes': ['Activities of Daily Living', 'Aged', 'Arm', 'Botulinum Toxins, Type A', 'Double-Blind Method', 'Female', 'Humans', 'Injections, Intramuscular', 'Male', 'Middle Aged', 'Motor Activity', 'Muscle Spasticity', 'Neuromuscular Agents', 'Randomized Controlled Trials as Topic', 'Stroke', 'Stroke Rehabilitation'], 'reasoning_required_pred': ['y', 'e', 's'], 'reasoning_free_pred': ['y', 'e', 's']}, 'long_answer': 'Using a targeted meta-analytic approach, it is possible to demonstrate that reducing spasticity in the arm is associated with a significant improvement in arm function.', 'final_decision': 'yes'}\n", "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 2 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n", "\u001b[32m2025-12-21 11:30:58.673\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n", "Example: {'pubid': 9542484, 'question': 'Does successful completion of the Perinatal Education Programme result in improved obstetric practice?', 'context': {'contexts': ['To determine whether successful completion of the Perinatal Education Programme (PEP) improves obstetric practice.', \"The three midwife obstetric units (MOUs) in a health district of Mpumalanga were included in the study. Two MOUs enrolled in the PEP and the third did not. A 'before-and-after' study design was used to assess any changes in practice, and to monitor whether any changes occurred in the district during the time of the study; data were also collected at the third MOU. Data were collected by scoring of the obstetric files after the patient had delivered.\", 'We ascertained whether the obstetric history, syphilis testing, blood group testing, haemoglobin measurement and uterine growth assessment were performed during antenatal care along with whether appropriate action was taken. For intrapartum care, estimation of fetal weight, the performance of pelvimetry, blood pressure monitoring, urine testing, evaluation of head above pelvis, fetal heart rate monitoring, monitoring of contractions and plotting of cervical dilatation, and whether the appropriate actions were taken, were assessed.', 'Eight of the 13 midwives at the two MOUs completed the PEP and all demonstrated an improvement in knowledge. Case notes of 303 patients from the various clinics were studied. There was no change in the referral patterns of any of the clinics during the study period. The obstetric history was well documented, but in no group was there a satisfactory response to a detected problem; appropriate action was taken in between 0% and 12% of cases. Syphilis testing was performed in 56-82% of cases, with no difference between the groups. The haemoglobin level was measured in only 4-15% of patients, with no difference before or after completion of the PEP. Where a problem in uterine growth was detected, an appropriate response occurred in 0-8% of patients and no difference before or after completion of the PEP was ascertained. In all groups, estimation of fetal weight and pelvimetry were seldom performed, the urine and fetal heart rate documentation were moderately well done and the blood pressure monitoring, assessment of head above pelvis, monitoring of contractions and plotting of cervical dilatation were usually performed. No differences before or after the PEP were detected. Where problems were detected, appropriate actions taken during labour improved, but not significantly.'], 'labels': ['OBJECTIVE', 'METHOD', 'OUTCOME MEASURES', 'RESULTS'], 'meshes': ['Africa', 'Female', 'Humans', 'Midwifery', 'Perinatal Care', 'Pregnancy', 'Prenatal Care', 'Rural Population'], 'reasoning_required_pred': ['n', 'o'], 'reasoning_free_pred': ['n', 'o']}, 'long_answer': 'Completion of the obstetric manual of the PEP improved the knowledge of the midwives but no alteration in practice was detected.', 'final_decision': 'no'}\n", "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 2 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n", "\u001b[32m2025-12-21 11:30:58.759\u001b[0m | \u001b[31m\u001b[1mERROR \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 2 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 37%|███▋ | 184/500 [00:38<02:35, 2.03it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "\u001b[32m2025-12-21 11:30:58.815\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n", "Example: {'pubid': 23228527, 'question': 'Does music influence stress in mechanically ventilated patients?', 'context': {'contexts': ['Mechanically ventilated patients experience profound stress. Interventions are needed to ameliorate stress that does not cause adverse effects. The purpose of this study was to explore the influence of music on stress in a sample of patients over the duration of ventilatory support.RESEARCH METHODOLOGY/', 'Randomised controlled trial; randomised patients (56.8+16.9 years, 61% male, APACHE III 57.2+18.3) receiving ventilatory support to: (1) patient-directed music (PDM) where patients self-initiated music listening whenever desired from a preferred collection, (2) headphones only to block ICU noise, or (3) usual ICU care. Twenty-four hour urinary cortisol samples were collected from a sub-set of subjects with intact renal function and not receiving medications known to influence cortisol levels (n=65).', '12 ICUs in the Midwestern United States.', 'Urinary free cortisol (UFC), an integrative biomarker of stress.', 'Controlling for illness severity, gender, and baseline UFC (29-45 mg/day), mixed models analysis revealed no significant differences among groups in UFC over the course of ventilatory support.'], 'labels': ['OBJECTIVES', 'DESIGN', 'SETTING', 'MAIN OUTCOME MEASURES', 'RESULTS'], 'meshes': ['Adult', 'Aged', 'Critical Illness', 'Female', 'Humans', 'Hydrocortisone', 'Male', 'Middle Aged', 'Music Therapy', 'Respiration, Artificial', 'Stress, Psychological'], 'reasoning_required_pred': ['n', 'o'], 'reasoning_free_pred': ['n', 'o']}, 'long_answer': 'While music did not significantly reduce cortisol, less profound spikes in UFC levels were observed but that, given the limitations of the research, this observation could have occurred merely by chance.', 'final_decision': 'no'}\n", "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 2 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n", "\u001b[32m2025-12-21 11:30:59.026\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n", "Example: {'pubid': 22211919, 'question': 'Actinobaculum schaalii, a cause of urinary tract infections in children?', 'context': {'contexts': ['Urine samples were examined by wet smear microscopy, incubated in 5% CO(2) for 1-2 days, and species-specific real-time polymerase chain reaction (PCR) for A. schaalii was performed.', 'In 5 of the 29 screened urines, A. schaalii was found only by real-time PCR in quantities equivalent to ≥ 10(4) -10(5) CFU/mL. In addition, A. schaalii was found in quantities equivalent to ≥ 10(6) CFU/mL by both culture and PCR in two children with a urinary tract infection and large numbers of leucocytes in the urine.'], 'labels': ['METHODS', 'RESULTS'], 'meshes': ['Adolescent', 'Child, Preschool', 'Female', 'Gram-Positive Bacteria', 'Gram-Positive Bacterial Infections', 'Humans', 'Infant', 'Male', 'Retrospective Studies', 'Urinary Tract Infections'], 'reasoning_required_pred': ['y', 'e', 's'], 'reasoning_free_pred': ['m', 'a', 'y', 'b', 'e']}, 'long_answer': 'Actinobaculum schaalii is CO(2)-dependent. Therefore, if there are clinical symptoms and/or a negative culture despite the presence of leucocytes in the urine, Gram staining and incubation in 5% CO(2) or species-specific real-time PCR should be performed to identify A. schaalii.', 'final_decision': 'maybe'}\n", "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n", "\u001b[32m2025-12-21 11:30:59.026\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n", "Example: {'pubid': 22720085, 'question': 'Does insulin resistance drive the association between hyperglycemia and cardiovascular risk?', 'context': {'contexts': ['Several studies have shown associations between hyperglycemia and risk of cardiovascular disease (CVD) and mortality, yet glucose-lowering treatment does little to mitigate this risk. We examined whether associations between hyperglycemia and CVD risk were explained by underlying insulin resistance.', 'In 60 middle-aged individuals without diabetes we studied the associations of fasting plasma glucose, 2-hour post oral glucose tolerance test plasma glucose, insulin sensitivity as well as body fat percentage with CVD risk. Insulin sensitivity was measured as the glucose infusion rate during a euglycemic hyperinsulinemic clamp, body fat percentage was measured by dual X-ray absorptiometry, and CVD risk was estimated using the Framingham risk score. Associations of fasting plasma glucose, 2-hour plasma glucose, insulin sensitivity and body fat percentage with the Framingham risk score were assessed in linear regression models.', 'Both fasting and 2-hour plasma glucose levels were associated with higher Framingham risk score (fasting glucose: r(2) = 0.21; 2-hour glucose: r(2) = 0.24; P<0.001 for both), and insulin sensitivity with lower Framingham risk score (r(2) = 0.36; P<0.001). However, adjustment for insulin sensitivity and 2-hour glucose made the effect of fasting glucose non-significant (P = 0.060). Likewise, when adjusting for insulin sensitivity and fasting glucose, the association between 2-hour glucose and Framingham risk score disappeared (P = 0.143). In contrast, insulin sensitivity was still associated with Framingham risk score after adjusting for glucose levels (P<0.001). Body fat was not associated with Framingham risk score when taking insulin sensitivity into account (P = 0.550).'], 'labels': ['BACKGROUND', 'METHODS', 'RESULTS'], 'meshes': ['Adult', 'Blood Glucose', 'Cardiovascular Diseases', 'Female', 'Humans', 'Hyperglycemia', 'Insulin Resistance', 'Male', 'Middle Aged', 'Risk Factors'], 'reasoning_required_pred': ['y', 'e', 's'], 'reasoning_free_pred': ['y', 'e', 's']}, 'long_answer': 'The association between plasma glucose levels and CVD risk is mainly explained by insulin resistance, which raises the question of whether glucose lowering per se without changes in the processes that underlie hyperglycemia should be the sole clinical paradigm in the treatment of type 2 diabetes or its prevention.', 'final_decision': 'yes'}\n", "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 37%|███▋ | 186/500 [00:38<01:56, 2.69it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m2025-12-21 11:30:59.139\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n", "Example: {'pubid': 17008699, 'question': 'Do patients with localized prostate cancer treatment really want more aggressive treatment?', 'context': {'contexts': ['Examine whether patients with prostate cancer choose the more aggressive of two radiotherapeutic options, whether this choice is reasoned, and what the determinants of the choice are.', 'One hundred fifty patients with primary prostate cancer (T(1-3)N(0)M(0)) were informed by means of a decision aid of two treatment options: radiotherapy with 70 Gy versus 74 Gy. The latter treatment is associated with more cure and more toxicity. The patients were asked whether they wanted to choose, and if so which treatment they preferred. They also assigned importance weights to the probability of various outcomes, such as survival, cure and adverse effects. Patients who wanted to choose their own treatment (n = 119) are described here.', 'The majority of these patients (75%) chose the lower radiation dose. Their choice was highly consistent (Por = 1.0 mm) was also positively related to hsCRP quartile, but this relationship was not significant after adjustment for age and other cardiovascular risk factors.'], 'labels': ['BACKGROUND', 'DESIGN', 'RESULTS'], 'meshes': ['Adult', 'Aged', 'Biomarkers', 'C-Reactive Protein', 'Carotid Artery Diseases', 'Carotid Artery, Common', 'Cross-Sectional Studies', 'Female', 'Humans', 'Korea', 'Male', 'Middle Aged', 'Nephelometry and Turbidimetry', 'Prevalence', 'Reference Values', 'Retrospective Studies', 'Risk Factors', 'Tunica Intima', 'Ultrasonography'], 'reasoning_required_pred': ['n', 'o'], 'reasoning_free_pred': ['n', 'o']}, 'long_answer': 'Both hsCRP levels and the carotid IMT were strongly correlated with conventional cardiovascular risk factors, but there was no independent association between hsCRP levels and carotid IMT in healthy Korean adults.', 'final_decision': 'no'}\n", "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 8 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 79%|███████▉ | 397/500 [01:40<00:59, 1.72it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m2025-12-21 11:32:00.855\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n", "Example: {'pubid': 25336163, 'question': 'Are interstitial fluid concentrations of meropenem equivalent to plasma concentrations in critically ill patients receiving continuous renal replacement therapy?', 'context': {'contexts': ['To describe the interstitial fluid (ISF) and plasma pharmacokinetics of meropenem in patients on continuous venovenous haemodiafiltration (CVVHDF).', 'This was a prospective observational pharmacokinetic study. Meropenem (500 mg) was administered every 8 h. CVVHDF was targeted as a 2-3 L/h exchange using a polyacrylonitrile filter with a surface area of 1.05 m2 and a blood flow rate of 200 mL/min. Serial blood (pre- and post-filter), filtrate/dialysate and ISF concentrations were measured on 2 days of treatment (Profiles A and B). Subcutaneous tissue ISF concentrations were determined using microdialysis.', 'A total of 384 samples were collected. During Profile A, the comparative median (IQR) ISF and plasma peak concentrations were 13.6 (12.0-16.8) and 40.7 (36.6-45.6) mg/L and the trough concentrations were 2.6 (2.4-3.4) and 4.9 (3.5-5.0) mg/L, respectively. During Profile B, the ISF trough concentrations increased by ∼40%. Meropenem ISF penetration was estimated at 63% (60%-69%) and 69% (65%-74%) for Profiles A and B, respectively, using comparative plasma and ISF AUCs. For Profile A, the plasma elimination t1/2 was 3.7 (3.3-4.0) h, the volume of distribution was 0.35 (0.25-0.46) L/kg, the total clearance was 4.1 (4.1-4.8) L/h and the CVVHDF clearance was 2.9 (2.7-3.1) L/h.'], 'labels': ['OBJECTIVES', 'PATIENTS AND METHODS', 'RESULTS'], 'meshes': ['Aged', 'Anti-Bacterial Agents', 'Area Under Curve', 'Critical Illness', 'Extracellular Fluid', 'Female', 'Hemodiafiltration', 'Humans', 'Intensive Care Units', 'Male', 'Middle Aged', 'Plasma', 'Prospective Studies', 'Renal Replacement Therapy', 'Thienamycins'], 'reasoning_required_pred': ['n', 'o'], 'reasoning_free_pred': ['n', 'o']}, 'long_answer': 'This is the first known report of concurrent plasma and ISF concentrations of a meropenem antibiotic during CVVHDF. We observed that the ISF concentrations of meropenem were significantly lower than the plasma concentrations, although the present dose was appropriate for infections caused by intermediately susceptible pathogens (MIC≤4 mg/L).', 'final_decision': 'no'}\n", "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 7 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n", "\u001b[32m2025-12-21 11:32:00.903\u001b[0m | \u001b[31m\u001b[1mERROR \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 7 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 80%|███████▉ | 399/500 [01:40<00:46, 2.16it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "\u001b[32m2025-12-21 11:32:00.915\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n", "Example: {'pubid': 11759976, 'question': 'Advanced epithelial ovarian carcinoma in Thai women: should we continue to offer second-look laparotomy?', 'context': {'contexts': ['To determine survival among patients with epithelial ovarian carcinoma (EOC) who underwent a second-look laparotomy (SLL) and those refusing the procedure. Also to analyze factor(s) influencing the survival of the patients.', 'Medical records were reviewed of patients with advanced EOC who were clinically free of disease after primary surgery and platinum-based chemotherapy between January 1, 1992, and December 31, 1998. All of them were offered SLL. Measurement outcomes include patient survival and disease-free survival.', \"There were 50 patients with clinically complete remission after chemotherapy. Sixteen patients underwent SLL, and thirty-four patients refused the procedure (NSLL). Seven patients (43.8%) were reported to have positive SLL. After the median follow-up time of 35 months, 12 patients had died, and 5 patients were lost to follow-up. The median survival time for patients with SLL was about 60 months. Five-year survival rates of patients in the SLL, and NSLL groups were 37 per cent (95%CI = 7%-69%), and 88 per cent (95%CI = 65%-96%) respectively (P<0.001). The median time to relapse was about 25 months for patients with negative SLL. Five-year disease-free survival rates of patients in the negative SLL, and NSLL groups were 28 per cent (95%CI = 4%-59%), and 54 per cent (95%CI = 34%-70%) respectively (P=0.251). By Cox regression analysis, tumor grade was the only significant prognostic factor influencing patients' survival (HR = 6, 95%CI of HR = 1.2-34.2).\"], 'labels': ['OBJECTIVE', 'METHOD AND MATERIAL', 'RESULTS'], 'meshes': ['Chemotherapy, Adjuvant', 'Disease-Free Survival', 'Female', 'Humans', 'Incidence', 'Neoplasm Recurrence, Local', 'Ovarian Neoplasms', 'Proportional Hazards Models', 'Second-Look Surgery', 'Survival Rate', 'Thailand'], 'reasoning_required_pred': ['m', 'a', 'y', 'b', 'e'], 'reasoning_free_pred': ['n', 'o']}, 'long_answer': \"The second-look laparotomy doesn't have a favorable impact on overall and disease-free survival. Tumor grade is the only independent prognostic variable for survival of the patients.\", 'final_decision': 'no'}\n", "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 7 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n", "\u001b[32m2025-12-21 11:32:00.917\u001b[0m | \u001b[31m\u001b[1mERROR \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 7 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "\u001b[32m2025-12-21 11:32:00.944\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n", "Example: {'pubid': 17105833, 'question': 'Empiric treatment of uncomplicated urinary tract infection with fluoroquinolones in older women in Israel: another lost treatment option?', 'context': {'contexts': ['Current guidelines for the treatment of uncomplicated urinary tract infection (UTI) in women recommend empiric therapy with antibiotics for which local resistance rates do not exceed 10-20%. We hypothesized that resistance rates of Escherichia coli to fluoroquinolones may have surpassed this level in older women in the Israeli community setting.', 'To identify age groups of women in which fluoroquinolones may no longer be appropriate for empiric treatment of UTI.', 'Resistance rates for ofloxacin were calculated for all cases of uncomplicated UTI diagnosed during the first 5 months of 2005 in a managed care organization (MCO) in Israel, in community-dwelling women aged 41-75 years. The women were without risk factors for fluoroquinolone resistance. Uncomplicated UTI was diagnosed with a urine culture positive for E. coli. The data set was stratified for age, using 5 year intervals, and stratum-specific resistance rates (% and 95% CI) were calculated. These data were analyzed to identify age groups in which resistance rates have surpassed 10%.', 'The data from 1291 urine cultures were included. The crude resistance rate to ofloxacin was 8.7% (95% CI 7.4 to 10.2). Resistance was lowest among the youngest (aged 41-50 y) women (3.2%; 95% CI 1.11 to 5.18), approached 10% in women aged 51-55 years (7.1%; 95% CI 3.4 to 10.9), and reached 19.86% (95% CI 13.2 to 26.5) among the oldest women (aged 56-75 y).'], 'labels': ['BACKGROUND', 'OBJECTIVES', 'METHODS', 'RESULTS'], 'meshes': ['Adolescent', 'Adult', 'Age Factors', 'Aged', 'Drug Resistance, Bacterial', 'Empirical Research', 'Escherichia coli', 'Escherichia coli Infections', 'Female', 'Fluoroquinolones', 'Humans', 'Israel', 'Middle Aged', 'Practice Guidelines as Topic', 'Urinary Tract Infections'], 'reasoning_required_pred': ['y', 'e', 's'], 'reasoning_free_pred': ['m', 'a', 'y', 'b', 'e']}, 'long_answer': 'Physicians who opt to treat UTI in postmenopausal women empirically should consider prescribing drugs other than fluoroquinolones. Concomitant longitudinal surveillance of both antibiotic utilization patterns and uropathogen resistance rates should become routine practice in this managed-care organization.', 'final_decision': 'maybe'}\n", "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 7 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n", "\u001b[32m2025-12-21 11:32:01.080\u001b[0m | \u001b[33m\u001b[1mWARNING \u001b[0m | \u001b[36mevoagentx.evaluators.evaluator\u001b[0m:\u001b[36m_evaluate_single_example\u001b[0m:\u001b[36m205\u001b[0m - \u001b[33m\u001b[1mError evaluating example and set the metrics to None:\n", "Example: {'pubid': 12947068, 'question': 'Do older men benefit from curative therapy of localized prostate cancer?', 'context': {'contexts': ['Prior decision-analytic models are based on outdated or suboptimal efficacy, patient preference, and comorbidity data. We estimated life expectancy (LE) and quality-adjusted life expectancy (QALE) associated with available treatments for localized prostate cancer in men aged>/= 65 years, adjusting for Gleason score, patient preferences, and comorbidity.', 'We evaluated three treatments, using a decision-analytic Markov model: radical prostatectomy (RP), external beam radiotherapy (EBRT), and watchful waiting (WW). Rates of treatment complications and pretreatment incontinence and impotence were derived from published studies. We estimated treatment efficacy using three data sources: cancer registry cohort data, pooled case series, and modern radiotherapy studies. Utilities were obtained from 141 prostate cancer patients and from published studies.', 'For men with well-differentiated tumors and few comorbidities, potentially curative therapy (RP or EBRT) prolonged LE up to age 75 years but did not improve QALE at any age. For moderately differentiated cancers, potentially curative therapy resulted in LE and QALE gains up to age 75 years. For poorly differentiated disease, potentially curative therapy resulted in LE and QALE gains up to age 80 years. Benefits of potentially curative therapy were restricted to men with no worse than mild comorbidity. When cohort and pooled case series data were used, RP was preferred over EBRT in all groups but was comparable to modern radiotherapy.'], 'labels': ['PURPOSE', 'METHODS', 'RESULTS'], 'meshes': ['Aged', 'Aged, 80 and over', 'Comorbidity', 'Decision Support Techniques', 'Humans', 'Life Expectancy', 'Male', 'Markov Chains', 'Postoperative Complications', 'Prostatectomy', 'Prostatic Neoplasms', 'Quality of Life', 'Radiotherapy', 'Treatment Outcome'], 'reasoning_required_pred': ['y', 'e', 's'], 'reasoning_free_pred': ['y', 'e', 's']}, 'long_answer': 'Potentially curative therapy results in significantly improved LE and QALE for older men with few comorbidities and moderately or poorly differentiated localized prostate cancer. Age should not be a barrier to treatment in this group.', 'final_decision': 'yes'}\n", "Error: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 7 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 81%|████████ | 403/500 [01:40<00:27, 3.49it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\u001b[32m2025-12-21 11:32:01.103\u001b[0m | \u001b[31m\u001b[1mERROR \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 7 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "\u001b[32m2025-12-21 11:32:01.127\u001b[0m | \u001b[31m\u001b[1mERROR \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 7 seconds. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 81%|████████ | 406/500 [01:48<01:30, 1.04it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 82%|████████▏ | 408/500 [02:00<03:21, 2.19s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 82%|████████▏ | 409/500 [02:01<02:54, 1.92s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "\u001b[32m2025-12-21 11:32:21.695\u001b[0m | \u001b[31m\u001b[1mERROR \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 82%|████████▏ | 411/500 [02:01<02:04, 1.40s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "\u001b[32m2025-12-21 11:32:21.764\u001b[0m | \u001b[31m\u001b[1mERROR \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 83%|████████▎ | 413/500 [02:01<01:30, 1.04s/it]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "\u001b[32m2025-12-21 11:32:22.064\u001b[0m | \u001b[31m\u001b[1mERROR \u001b[0m | \u001b[36mevoagentx.workflow.workflow\u001b[0m:\u001b[36masync_execute\u001b[0m:\u001b[36m104\u001b[0m - \u001b[31m\u001b[1mAn Error occurs when executing the workflow: Error during single_generate_async: litellm.RateLimitError: AzureException RateLimitError - Your requests to gpt-4o-mini for gpt-4o-mini in East US 2 have exceeded the token rate limit for your current AIServices S0 pricing tier. This request was for ChatCompletions_Create under Azure OpenAI API version 2025-01-01-preview. Please retry after 1 second. To increase your default rate limit, visit: https://aka.ms/oai/quotaincrease.\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 83%|████████▎ | 415/500 [02:01<01:03, 1.33it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 83%|████████▎ | 417/500 [02:03<01:00, 1.37it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 84%|████████▎ | 418/500 [02:03<00:52, 1.57it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 84%|████████▍ | 420/500 [02:03<00:36, 2.20it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 85%|████████▍ | 424/500 [02:03<00:15, 4.75it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 85%|████████▌ | 426/500 [02:04<00:14, 5.08it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 86%|████████▋ | 432/500 [02:04<00:09, 7.10it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 87%|████████▋ | 434/500 [02:05<00:08, 7.51it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 87%|████████▋ | 436/500 [02:05<00:07, 8.02it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 88%|████████▊ | 438/500 [02:05<00:09, 6.61it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 88%|████████▊ | 440/500 [02:06<00:10, 5.59it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 89%|████████▊ | 443/500 [02:06<00:07, 7.79it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 90%|████████▉ | 448/500 [02:07<00:07, 7.27it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 90%|█████████ | 451/500 [02:07<00:06, 7.84it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 91%|█████████ | 453/500 [02:07<00:06, 7.25it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 91%|█████████ | 455/500 [02:08<00:07, 6.13it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 91%|█████████ | 456/500 [02:08<00:08, 5.24it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 92%|█████████▏| 458/500 [02:09<00:07, 5.44it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 92%|█████████▏| 462/500 [02:09<00:05, 6.64it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 93%|█████████▎| 464/500 [02:09<00:04, 7.87it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 93%|█████████▎| 467/500 [02:10<00:04, 7.03it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 94%|█████████▍| 469/500 [02:10<00:03, 8.73it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 94%|█████████▍| 472/500 [02:10<00:03, 8.70it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 95%|█████████▍| 474/500 [02:11<00:04, 6.35it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 95%|█████████▌| 475/500 [02:11<00:04, 5.81it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 95%|█████████▌| 477/500 [02:11<00:03, 6.92it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 96%|█████████▌| 479/500 [02:11<00:02, 7.06it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 96%|█████████▌| 480/500 [02:12<00:04, 4.14it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 97%|█████████▋| 483/500 [02:12<00:02, 6.26it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 97%|█████████▋| 485/500 [02:13<00:02, 5.45it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 97%|█████████▋| 487/500 [02:13<00:01, 6.58it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 98%|█████████▊| 489/500 [02:13<00:01, 7.31it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 98%|█████████▊| 491/500 [02:13<00:01, 7.89it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 98%|█████████▊| 492/500 [02:14<00:01, 6.50it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 99%|█████████▉| 494/500 [02:14<00:01, 5.64it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 0.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 99%|█████████▉| 496/500 [02:14<00:00, 7.92it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", "Evaluating workflow: 100%|█████████▉| 498/500 [02:14<00:00, 7.38it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Evaluating workflow: 100%|██████████| 500/500 [02:15<00:00, 3.69it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "metrics {'f1': 0, 'em': 0.0, 'acc': 1.0}\n", "Evaluation metrics: {'f1': 0.0, 'em': 0.0, 'acc': 0.635593220338983}\n", "\u001b[32m2025-12-21 11:32:36.018\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.workflow.workflow_graph\u001b[0m:\u001b[36msave_module\u001b[0m:\u001b[36m1201\u001b[0m - \u001b[1mSaving SEWWorkFlowGraph to debug/optimized_sew_workflow_update_correct_round20_step20_gpt4omini_optzall.json\u001b[0m\n", "\u001b[32m2025-12-21 11:32:36.018\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mevoagentx.utils.utils\u001b[0m:\u001b[36mmake_parent_folder\u001b[0m:\u001b[36m19\u001b[0m - \u001b[1mcreating folder debug ...\u001b[0m\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n" ] } ], "source": [ "# obtain SEWOptimizer after having more roles\n", "optimizer = SEWOptimizer(\n", " graph=sew_graph, \n", " evaluator=evaluator, \n", " llm=llm, \n", " max_steps=20,\n", " eval_rounds=1, \n", " repr_scheme=\"python\", \n", " optimize_mode=\"all\", \n", " order=\"zero-order\",\n", " max_rounds=20,\n", ")\n", "optimizer.optimize(dataset=benchmark)\n", "with suppress_logger_info():\n", " metrics = optimizer.evaluate(dataset=benchmark, eval_mode=\"test\")\n", "print(\"Evaluation metrics: \", metrics)\n", "# save the optimized SEW workflow\n", "optimizer.save(\"debug/optimized_sew_workflow_update_correct_round20_step20_gpt4omini_optzall.json\")" ] }, { "cell_type": "code", "execution_count": 29, "id": "c5f272e3", "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'f1': 0.0, 'em': 0.0, 'acc': 0.635593220338983}" ] }, "execution_count": 29, "metadata": {}, "output_type": "execute_result" } ], "source": [ "metrics" ] }, { "cell_type": "code", "execution_count": null, "id": "3e2dc63b", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.11.13" } }, "nbformat": 4, "nbformat_minor": 5 }